Add support for SQLite3 full-text-search and other virtual tables.

Previously, adding sqlite3 virtual tables messed up `schema.rb`.

Now, virtual tables can safely be added using `create_virtual_table`.

activerecord/CHANGELOG.md

@@ -1,3 +1,11 @@
+*   Add support for SQLite3 full-text-search and other virtual tables.
+
+    Previously, adding sqlite3 virtual tables messed up `schema.rb`.
+
+    Now, virtual tables can safely be added using `create_virtual_table`.
+
+    *Zacharias Knudsen*
+
 *   Support use of alternative database interfaces via the `database_cli` ActiveRecord configuration option.
 
     ```ruby

activerecord/lib/active_record/connection_adapters/abstract_adapter.rb

@@ -595,6 +595,14 @@ module ActiveRecord
       def rename_enum_value(*) # :nodoc:
       end
 
+      # This is meant to be implemented by the adapters that support virtual tables
+      def create_virtual_table(*) # :nodoc:
+      end
+
+      # This is meant to be implemented by the adapters that support virtual tables
+      def drop_virtual_table(*) # :nodoc:
+      end
+
       def advisory_locks_enabled? # :nodoc:
         supports_advisory_locks? && @advisory_locks_enabled
       end

activerecord/lib/active_record/connection_adapters/sqlite3/schema_dumper.rb

@@ -5,6 +5,19 @@ module ActiveRecord
     module SQLite3
       class SchemaDumper < ConnectionAdapters::SchemaDumper # :nodoc:
         private
+          def virtual_tables(stream)
+            virtual_tables = @connection.virtual_tables
+            if virtual_tables.any?
+              stream.puts
+              stream.puts "  # Virtual tables defined in this database."
+              stream.puts "  # Note that virtual tables may not work with other database engines. Be careful if changing database."
+              virtual_tables.sort.each do |table_name, options|
+                module_name, arguments = options
+                stream.puts "  create_virtual_table #{table_name.inspect}, #{module_name.inspect}, #{arguments.split(", ").inspect}"
+              end
+            end
+          end
+
           def default_primary_key?(column)
             schema_type(column) == :integer
           end

activerecord/lib/active_record/connection_adapters/sqlite3/schema_statements.rb

@@ -82,6 +82,10 @@ module ActiveRecord
           alter_table(from_table, foreign_keys)
         end
 
+        def virtual_table_exists?(table_name)
+          query_values(data_source_sql(table_name, type: "VIRTUAL TABLE"), "SCHEMA").any?
+        end
+
         def check_constraints(table_name)
           table_sql = query_value(<<-SQL, "SCHEMA")
             SELECT sql
@@ -176,7 +180,8 @@ module ActiveRecord
             scope = quoted_scope(name, type: type)
             scope[:type] ||= "'table','view'"
 
-            sql = +"SELECT name FROM sqlite_master WHERE name <> 'sqlite_sequence'"
+            sql = +"SELECT name FROM pragma_table_list WHERE schema <> 'temp'"
+            sql << " AND name NOT IN ('sqlite_sequence', 'sqlite_schema')"
             sql << " AND name = #{scope[:name]}" if scope[:name]
             sql << " AND type IN (#{scope[:type]})"
             sql
@@ -189,6 +194,8 @@ module ActiveRecord
                 "'table'"
               when "VIEW"
                 "'view'"
+              when "VIRTUAL TABLE"
+                "'virtual'"
               end
             scope = {}
             scope[:name] = quote(name) if name

activerecord/lib/active_record/connection_adapters/sqlite3_adapter.rb

@@ -283,6 +283,38 @@ module ActiveRecord
         exec_query "DROP INDEX #{quote_column_name(index_name)}"
       end
 
+      VIRTUAL_TABLE_REGEX = /USING\s+(\w+)\s*\((.+)\)/i
+
+      # Returns a list of defined virtual tables
+      def virtual_tables
+        query = <<~SQL
+          SELECT name, sql FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL %';
+        SQL
+
+        exec_query(query, "SCHEMA").cast_values.each_with_object({}) do |row, memo|
+          table_name, sql = row[0], row[1]
+          _, module_name, arguments = sql.match(VIRTUAL_TABLE_REGEX).to_a
+          memo[table_name] = [module_name, arguments]
+        end.to_a
+      end
+
+      # Creates a virtual table
+      #
+      # Example:
+      #   create_virtual_table :emails, :fts5, ['sender', 'title',' body']
+      def create_virtual_table(table_name, module_name, values)
+        exec_query "CREATE VIRTUAL TABLE IF NOT EXISTS #{table_name} USING #{module_name} (#{values.join(", ")})"
+      end
+
+      # Drops a virtual table
+      #
+      # Although this command ignores +module_name+ and +values+,
+      # it can be helpful to provide these in a migration's +change+ method so it can be reverted.
+      # In that case, +module_name+, +values+ and +options+ will be used by #create_virtual_table.
+      def drop_virtual_table(table_name, module_name, values, **options)
+        drop_table(table_name)
+      end
+
       # Renames a table.
       #
       # Example:

activerecord/lib/active_record/migration/command_recorder.rb

@@ -22,10 +22,12 @@ module ActiveRecord
     # * change_table_comment (must supply a +:from+ and +:to+ option)
     # * create_enum
     # * create_join_table
+    # * create_virtual_table
     # * create_table
     # * disable_extension
     # * drop_enum (must supply a list of values)
     # * drop_join_table
+    # * drop_virtual_table (must supply options)
     # * drop_table (must supply a block)
     # * enable_extension
     # * remove_column (must supply a type)
@@ -56,6 +58,7 @@ module ActiveRecord
         :add_unique_constraint, :remove_unique_constraint,
         :create_enum, :drop_enum, :rename_enum, :add_enum_value, :rename_enum_value,
         :create_schema, :drop_schema,
+        :create_virtual_table, :drop_virtual_table
       ]
       include JoinTable
 
@@ -166,6 +169,7 @@ module ActiveRecord
               enable_extension:  :disable_extension,
               create_enum:       :drop_enum,
               create_schema:     :drop_schema,
+              create_virtual_table: :drop_virtual_table
             }.each do |cmd, inv|
               [[inv, cmd], [cmd, inv]].uniq.each do |method, inverse|
                 class_eval <<-EOV, __FILE__, __LINE__ + 1
@@ -374,6 +378,12 @@ module ActiveRecord
           [:rename_enum_value, [type_name, from: options[:to], to: options[:from]]]
         end
 
+        def invert_drop_virtual_table(args)
+          _enum, values = args.dup.tap(&:extract_options!)
+          raise ActiveRecord::IrreversibleMigration, "drop_virtual_table is only reversible if given options." unless values
+          super
+        end
+
         def respond_to_missing?(method, _)
           super || delegate.respond_to?(method)
         end

activerecord/lib/active_record/schema_dumper.rb

@@ -63,6 +63,7 @@ module ActiveRecord
       extensions(stream)
       types(stream)
       tables(stream)
+      virtual_tables(stream)
       trailer(stream)
       stream
     end
@@ -126,6 +127,10 @@ module ActiveRecord
       def schemas(stream)
       end
 
+      # virtual tables are only supported by SQLite
+      def virtual_tables(stream)
+      end
+
       def tables(stream)
         sorted_tables = @connection.tables.sort
 

activerecord/test/cases/adapters/sqlite3/sqlite3_adapter_test.rb

@@ -596,7 +596,7 @@ module ActiveRecord
 
       def test_tables_logs_name
         sql = <<~SQL
-          SELECT name FROM sqlite_master WHERE name <> 'sqlite_sequence' AND type IN ('table')
+          SELECT name FROM pragma_table_list WHERE schema <> 'temp' AND name NOT IN ('sqlite_sequence', 'sqlite_schema') AND type IN ('table')
         SQL
         @conn.connect!
         assert_logged [[sql.squish, "SCHEMA", []]] do
@@ -607,7 +607,7 @@ module ActiveRecord
       def test_table_exists_logs_name
         with_example_table do
           sql = <<~SQL
-            SELECT name FROM sqlite_master WHERE name <> 'sqlite_sequence' AND name = 'ex' AND type IN ('table')
+            SELECT name FROM pragma_table_list WHERE schema <> 'temp' AND name NOT IN ('sqlite_sequence', 'sqlite_schema') AND name = 'ex' AND type IN ('table')
           SQL
           assert_logged [[sql.squish, "SCHEMA", []]] do
             assert @conn.table_exists?("ex")

activerecord/test/cases/adapters/sqlite3/virtual_table_test.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require "cases/helper"
+require "support/schema_dumping_helper"
+
+class SQLite3VirtualTableTest < ActiveRecord::SQLite3TestCase
+  include SchemaDumpingHelper
+
+  def setup
+    @connection = ActiveRecord::Base.lease_connection
+    @connection.create_virtual_table :searchables, :fts5, ["content", "meta UNINDEXED", "tokenize='porter ascii'"]
+  end
+
+  def teardown
+    @connection.drop_table :searchables, if_exists: true
+  end
+
+  def test_schema_dump
+    output = dump_all_table_schema
+
+    assert_not_includes output, "searchables_docsize"
+    assert_includes output, 'create_virtual_table "searchables", "fts5", ["content", "meta UNINDEXED", "tokenize=\'porter ascii\'"]'
+  end
+
+  def test_schema_load
+    original, $stdout = $stdout, StringIO.new
+
+    ActiveRecord::Schema.define do
+      create_virtual_table :emails, :fts5, ["content", "meta UNINDEXED"]
+    end
+
+    assert @connection.virtual_table_exists?(:emails)
+  ensure
+    $stdout = original
+  end
+end

activerecord/test/cases/migration/command_recorder_test.rb

@@ -564,6 +564,22 @@ module ActiveRecord
           @recorder.inverse_of :rename_enum_value, [:dog_breed, from: :beagle]
         end
       end
+
+      def test_invert_create_virtual_table
+        drop = @recorder.inverse_of :create_virtual_table, [:searchables, :fts5, ["content", "meta UNINDEXED", "tokenize='porter ascii'"]]
+        assert_equal [:drop_virtual_table, [:searchables, :fts5, ["content", "meta UNINDEXED", "tokenize='porter ascii'"]], nil], drop
+      end
+
+      def test_invert_drop_virtual_table
+        create = @recorder.inverse_of :drop_virtual_table, [:searchables, :fts5, ["title", "content"]]
+        assert_equal [:create_virtual_table, [:searchables, :fts5, ["title", "content"]], nil], create
+      end
+
+      def test_invert_drop_virtual_table_without_options
+        assert_raises(ActiveRecord::IrreversibleMigration) do
+          @recorder.inverse_of :drop_virtual_table, [:searchables]
+        end
+      end
     end
   end
 end

railties/test/application/bin_setup_test.rb

@@ -13,7 +13,7 @@ module ApplicationTests
       Dir.chdir(app_path) do
         rails "generate", "model", "article"
 
-        list_tables = lambda { rails("runner", "p ActiveRecord::Base.lease_connection.tables").strip }
+        list_tables = lambda { rails("runner", "p ActiveRecord::Base.lease_connection.tables.sort").strip }
         File.write("log/test.log", "zomg!")
 
         assert_equal "[]", list_tables.call
@@ -22,7 +22,7 @@ module ApplicationTests
 
         `bin/setup 2>&1`
         assert_equal 0, File.size("log/test.log")
-        assert_equal '["schema_migrations", "ar_internal_metadata", "articles"]', list_tables.call
+        assert_equal '["ar_internal_metadata", "articles", "schema_migrations"]', list_tables.call
         assert File.exist?("tmp/restart.txt")
       end
     end

railties/test/application/rake/dbs_test.rb

@@ -541,11 +541,11 @@ module ApplicationTests
           end
         RUBY
 
-        list_tables = lambda { rails("runner", "p ActiveRecord::Base.lease_connection.tables").strip }
+        list_tables = lambda { rails("runner", "p ActiveRecord::Base.lease_connection.tables.sort").strip }
 
         assert_equal '["posts"]', list_tables[]
         rails "db:schema:load"
-        assert_equal '["posts", "comments", "schema_migrations", "ar_internal_metadata"]', list_tables[]
+        assert_equal '["ar_internal_metadata", "comments", "posts", "schema_migrations"]', list_tables[]
 
         add_to_config "config.active_record.schema_format = :sql"
         app_file "db/structure.sql", <<-SQL
@@ -553,7 +553,7 @@ module ApplicationTests
         SQL
 
         rails "db:schema:load"
-        assert_equal '["posts", "comments", "schema_migrations", "ar_internal_metadata", "users"]', list_tables[]
+        assert_equal '["ar_internal_metadata", "comments", "posts", "schema_migrations", "users"]', list_tables[]
       end
 
       test "db:schema:load with inflections" do

railties/test/application/rake/multi_dbs_test.rb

@@ -108,11 +108,11 @@ module ApplicationTests
 
           rails "db:schema:load"
 
-          ar_tables = lambda { rails("runner", "p ActiveRecord::Base.lease_connection.tables").strip }
-          animals_tables = lambda { rails("runner", "p AnimalsBase.lease_connection.tables").strip }
+          ar_tables = lambda { rails("runner", "p ActiveRecord::Base.lease_connection.tables.sort").strip }
+          animals_tables = lambda { rails("runner", "p AnimalsBase.lease_connection.tables.sort").strip }
 
-          assert_equal '["schema_migrations", "ar_internal_metadata", "books"]', ar_tables[]
-          assert_equal '["schema_migrations", "ar_internal_metadata", "dogs"]', animals_tables[]
+          assert_equal '["ar_internal_metadata", "books", "schema_migrations"]', ar_tables[]
+          assert_equal '["ar_internal_metadata", "dogs", "schema_migrations"]', animals_tables[]
         end
       end
 
@@ -148,15 +148,15 @@ module ApplicationTests
 
           rails "db:schema:load:#{database}"
 
-          ar_tables = lambda { rails("runner", "p ActiveRecord::Base.lease_connection.tables").strip }
-          animals_tables = lambda { rails("runner", "p AnimalsBase.lease_connection.tables").strip }
+          ar_tables = lambda { rails("runner", "p ActiveRecord::Base.lease_connection.tables.sort").strip }
+          animals_tables = lambda { rails("runner", "p AnimalsBase.lease_connection.tables.sort").strip }
 
           if database == "primary"
-            assert_equal '["schema_migrations", "ar_internal_metadata", "books"]', ar_tables[]
+            assert_equal '["ar_internal_metadata", "books", "schema_migrations"]', ar_tables[]
             assert_equal "[]", animals_tables[]
           else
             assert_equal "[]", ar_tables[]
-            assert_equal '["schema_migrations", "ar_internal_metadata", "dogs"]', animals_tables[]
+            assert_equal '["ar_internal_metadata", "dogs", "schema_migrations"]', animals_tables[]
           end
         end
       end
@@ -211,15 +211,15 @@ module ApplicationTests
           output = rails("db:test:prepare:#{name}", "--trace")
           assert_match(/Execute db:test:load_schema:#{name}/, output)
 
-          ar_tables = lambda { rails("runner", "-e", "test", "p ActiveRecord::Base.lease_connection.tables").strip }
-          animals_tables = lambda { rails("runner",  "-e", "test", "p AnimalsBase.lease_connection.tables").strip }
+          ar_tables = lambda { rails("runner", "-e", "test", "p ActiveRecord::Base.lease_connection.tables.sort").strip }
+          animals_tables = lambda { rails("runner",  "-e", "test", "p AnimalsBase.lease_connection.tables.sort").strip }
 
           if name == "primary"
-            assert_equal ["schema_migrations", "ar_internal_metadata", "books"].sort, JSON.parse(ar_tables[]).sort
+            assert_equal '["ar_internal_metadata", "books", "schema_migrations"]', ar_tables[]
             assert_equal "[]", animals_tables[]
           else
             assert_equal "[]", ar_tables[]
-            assert_equal ["schema_migrations", "ar_internal_metadata", "dogs"].sort, JSON.parse(animals_tables[]).sort
+            assert_equal '["ar_internal_metadata", "dogs", "schema_migrations"]', animals_tables[]
           end
         end
       end