refactor and improve partition management
* move most of the logic into PartitionManager * use a setting for how many partitions to precreate/prune * enumerate existing tables for pruning, instead of assuming we can just roll off the last one * actually run the tests for partman from CI * simplify configuring partitioned migrations (just rely on the model to get the table name, instead of trying to guess the model from the table name, and sometimes having to provide both) Change-Id: Ic8ac2b603a02f092b2f278d2b366b1cd9f942954 Reviewed-on: https://gerrit.instructure.com/53286 Tested-by: Jenkins Reviewed-by: Ethan Vizitei <evizitei@instructure.com> Reviewed-by: Brian Finney <bfinney@instructure.com> Product-Review: Cody Cutrer <cody@instructure.com> QA-Review: Cody Cutrer <cody@instructure.com>
This commit is contained in:
Cody Cutrer
parent
157ee9bfc6
commit
be8b9e0772
|
|
@ -1,4 +1,4 @@
|
|||
class Quizzes::QuizSubmissionEventPartitioner < ActiveRecord::Base
|
||||
class Quizzes::QuizSubmissionEventPartitioner
|
||||
cattr_accessor :logger
|
||||
|
||||
def self.process
|
||||
|
|
@ -8,29 +8,9 @@ class Quizzes::QuizSubmissionEventPartitioner < ActiveRecord::Base
|
|||
|
||||
partman = CanvasPartman::PartitionManager.new(Quizzes::QuizSubmissionEvent)
|
||||
|
||||
[ Time.now.utc, 1.month.from_now(Time.now.utc) ].each do |date|
|
||||
log "Looking for a table for partition #{date.strftime('%Y/%m')}..."
|
||||
partman.ensure_partitions(Setting.get('quiz_events_partitions_precreate_months', 2).to_i)
|
||||
|
||||
if partman.partition_exists?(date)
|
||||
log "\tPartition table exists, nothing to do. [OK]"
|
||||
else
|
||||
log "\tPartition table does not exist, creating..."
|
||||
partition_table_name = partman.create_partition(date)
|
||||
log "\tPartition table created: '#{partition_table_name}'. [OK]"
|
||||
end
|
||||
end
|
||||
|
||||
# on 5/1, we want to drop 10/1
|
||||
# (keeping 11, 12, 1, 2, 3, and 4 - 6 months of data)
|
||||
[ 7.months.ago(Time.now.utc.beginning_of_month) ].each do |date|
|
||||
log "Looking for old partition table (#{date.strftime('%Y/%m')})..."
|
||||
|
||||
if partman.partition_exists?(date)
|
||||
log "\tPartition table exists, dropping..."
|
||||
partman.drop_partition(date)
|
||||
log "\tPartition table dropped. [OK]"
|
||||
end
|
||||
end
|
||||
partman.prune_partitions(Setting.get("quiz_events_partitions_keep_months", 6).to_i)
|
||||
|
||||
log 'Done. Bye!'
|
||||
log '*' * 80
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
class AddIndicesToQuizSubmissionEvents < CanvasPartman::Migration
|
||||
tag :predeploy
|
||||
|
||||
self.master_table = :quiz_submission_events
|
||||
self.base_class = Quizzes::QuizSubmissionEvent
|
||||
|
||||
def up
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ platforms :ruby_19 do
|
|||
gem 'debugger'
|
||||
end
|
||||
|
||||
platforms :ruby_21 do
|
||||
platforms :ruby_21, :ruby_22 do
|
||||
gem 'byebug', require: false
|
||||
end
|
||||
|
||||
|
|
|
|||
|
|
@ -4,19 +4,21 @@ require 'canvas_partman/dynamic_relation'
|
|||
require 'canvas_partman/concerns/partitioned'
|
||||
|
||||
module CanvasPartman
|
||||
# @property [String, "db/migrate"] migrations_paths
|
||||
# Path (relative to Rails root) to where the partition migrations should be
|
||||
# looked up.
|
||||
mattr_accessor :migrations_path
|
||||
class << self
|
||||
# @property [String, "db/migrate"] migrations_paths
|
||||
# Path (relative to Rails root) to where the partition migrations should be
|
||||
# looked up.
|
||||
attr_accessor :migrations_path
|
||||
|
||||
# @property [String, "partitions"] migrations_scope
|
||||
# The filename "scope" that identifies partition migrations. This is a key
|
||||
# that is separated from the name of the migration file and the "rb"
|
||||
# extension by dots.
|
||||
#
|
||||
# Example: "partitions" => "20141215000000_add_something.partitions.rb"
|
||||
mattr_accessor :migrations_scope
|
||||
# @property [String, "partitions"] migrations_scope
|
||||
# The filename "scope" that identifies partition migrations. This is a key
|
||||
# that is separated from the name of the migration file and the "rb"
|
||||
# extension by dots.
|
||||
#
|
||||
# Example: "partitions" => "20141215000000_add_something.partitions.rb"
|
||||
attr_accessor :migrations_scope
|
||||
end
|
||||
|
||||
self.migrations_path = 'db/migrate'
|
||||
self.migrations_scope = 'partitions'
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -1,52 +1,43 @@
|
|||
require 'active_record/migration'
|
||||
|
||||
class CanvasPartman::Migration < ActiveRecord::Migration
|
||||
class << self
|
||||
# @attr [String] master_table
|
||||
# Name of the master table which partitions this migration will modify.
|
||||
attr_accessor :master_table
|
||||
module CanvasPartman
|
||||
class Migration < ActiveRecord::Migration
|
||||
class << self
|
||||
# @attr [CanvasPartman::Partitioned] base_class
|
||||
# The partitioned ActiveRecord::Base model _class_ we're modifying
|
||||
attr_accessor :base_class
|
||||
end
|
||||
|
||||
# @attr [CanvasPartman::Partitioned] base_class
|
||||
# The partitioned ActiveRecord::Base model _class_ we're modifying (which is
|
||||
# stored in the master_table we specified earlier.)
|
||||
def with_each_partition(&block)
|
||||
find_partition_tables.each(&block)
|
||||
end
|
||||
|
||||
# Bind the migration to apply only on a certain partition table. Routines
|
||||
# like #with_each_partition will yield only the specified table instead.
|
||||
#
|
||||
# If left unspecified, we try to infer the class from the master table name.
|
||||
attr_accessor :base_class
|
||||
# @param [String] table_name
|
||||
# Name of the **existing** partition table.
|
||||
def restrict_to_partition(table_name)
|
||||
@partition_scope = table_name
|
||||
yield
|
||||
ensure
|
||||
@partition_scope = nil
|
||||
end
|
||||
|
||||
# :nodoc:
|
||||
attr_reader :partition_table_matcher
|
||||
def self.connection
|
||||
(base_class || ActiveRecord::Base).connection
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def partition_manager
|
||||
@partition_manager ||= PartitionManager.new(self.class.base_class)
|
||||
end
|
||||
|
||||
def find_partition_tables
|
||||
return [ @partition_scope ] if @partition_scope
|
||||
|
||||
partition_manager.partition_tables
|
||||
end
|
||||
end
|
||||
|
||||
def with_each_partition(&block)
|
||||
find_partition_tables.each(&block)
|
||||
end
|
||||
|
||||
# Bind the migration to apply only on a certain partition table. Routines
|
||||
# like #with_each_partition will yield only the specified table instead.
|
||||
#
|
||||
# @param [String] table_name
|
||||
# Name of the **existing** partition table.
|
||||
def restrict_to_partition(table_name)
|
||||
@partition_scope = table_name
|
||||
yield
|
||||
ensure
|
||||
@partition_scope = nil
|
||||
end
|
||||
|
||||
def self.connection
|
||||
(self.base_class || ActiveRecord::Base).connection
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def self.master_table=(name)
|
||||
@master_table = name.to_s.freeze
|
||||
@partition_table_matcher = /^#{Regexp.escape(@master_table)}_/.freeze
|
||||
end
|
||||
|
||||
def find_partition_tables
|
||||
return [ @partition_scope ] if @partition_scope
|
||||
|
||||
self.class.connection.tables.grep(self.class.partition_table_matcher)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -13,6 +13,36 @@ module CanvasPartman
|
|||
@base_class = base_class
|
||||
end
|
||||
|
||||
# Ensure the current partition, and n future partitions exist
|
||||
#
|
||||
# @param [Fixnum] advance
|
||||
# The number of partitions to create in advance
|
||||
def ensure_partitions(advance = 1)
|
||||
current = Time.now.utc.send("beginning_of_#{base_class.partitioning_interval.to_s.singularize}")
|
||||
(advance + 1).times do
|
||||
unless partition_exists?(current)
|
||||
create_partition(current)
|
||||
end
|
||||
current += 1.send(base_class.partitioning_interval)
|
||||
end
|
||||
end
|
||||
|
||||
# Prune old partitions
|
||||
#
|
||||
# @param [Fixnum] number_to_keep
|
||||
# The number of partitions to keep (excluding the current partition)
|
||||
def prune_partitions(number_to_keep = 6)
|
||||
min_to_keep = Time.now.utc.send("beginning_of_#{base_class.partitioning_interval.to_s.singularize}")
|
||||
# on 5/1, we want to drop 10/1
|
||||
# (keeping 11, 12, 1, 2, 3, and 4 - 6 months of data)
|
||||
min_to_keep -= number_to_keep.send(base_class.partitioning_interval)
|
||||
|
||||
partition_tables.each do |table|
|
||||
partition_date = date_from_partition_name(table)
|
||||
base_class.connection.drop_table(table) if partition_date < min_to_keep
|
||||
end
|
||||
end
|
||||
|
||||
# Create a new partition table.
|
||||
#
|
||||
# @param [Hash] options
|
||||
|
|
@ -47,7 +77,7 @@ module CanvasPartman
|
|||
INHERITS (#{master_table});
|
||||
SQL
|
||||
|
||||
find_and_load_migrations(master_table).each do |migration|
|
||||
find_and_load_migrations.each do |migration|
|
||||
migration.restrict_to_partition(partition_table) do
|
||||
migration.migrate(:up)
|
||||
end
|
||||
|
|
@ -57,6 +87,10 @@ module CanvasPartman
|
|||
end
|
||||
end
|
||||
|
||||
def partition_tables
|
||||
base_class.connection.tables.grep(table_regex)
|
||||
end
|
||||
|
||||
def partition_exists?(date_or_name)
|
||||
table_name = if date_or_name.kind_of?(Time)
|
||||
generate_name_for_partition(date_or_name)
|
||||
|
|
@ -75,6 +109,15 @@ module CanvasPartman
|
|||
|
||||
protected
|
||||
|
||||
def table_regex
|
||||
@table_regex ||= case base_class.partitioning_interval
|
||||
when :months
|
||||
/^#{Regexp.escape(base_class.table_name)}_(?<year>\d{4,})_(?<month>\d{1,2})$/.freeze
|
||||
when :years
|
||||
/^#{Regexp.escape(base_class.table_name)}_(?<year>\d{4,})$/.freeze
|
||||
end
|
||||
end
|
||||
|
||||
def generate_name_for_partition(date)
|
||||
date_attr = Arel::Attributes::Attribute.new(nil, base_class.partitioning_field)
|
||||
|
||||
|
|
@ -84,6 +127,12 @@ module CanvasPartman
|
|||
base_class.infer_partition_table_name(attributes)
|
||||
end
|
||||
|
||||
def date_from_partition_name(name)
|
||||
match = table_regex.match(name)
|
||||
return nil unless match
|
||||
Time.utc(*match[1..-1])
|
||||
end
|
||||
|
||||
def generate_date_constraint_range(date)
|
||||
case base_class.partitioning_interval
|
||||
when :months
|
||||
|
|
@ -104,14 +153,14 @@ module CanvasPartman
|
|||
end
|
||||
end
|
||||
|
||||
def find_and_load_migrations(master_table)
|
||||
def find_and_load_migrations
|
||||
ActiveRecord::Migrator.migrations(CanvasPartman.migrations_path).reduce([]) do |migrations, proxy|
|
||||
if proxy.scope == CanvasPartman.migrations_scope
|
||||
require(File.expand_path(proxy.filename))
|
||||
|
||||
migration_klass = proxy.name.constantize
|
||||
|
||||
if migration_klass.master_table == master_table
|
||||
if migration_klass.base_class == base_class
|
||||
migrations << migration_klass.new
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -8,15 +8,15 @@ class PartitionMigrationGenerator < ActiveRecord::Generators::MigrationGenerator
|
|||
source_root File.expand_path("../templates", __FILE__)
|
||||
|
||||
remove_argument :attributes
|
||||
argument :master_table, type: :string, required: false,
|
||||
desc: 'Name of the master table whose partitions will be modified.'
|
||||
argument :model, type: :string, required: false,
|
||||
desc: 'Name of the model whose partitions will be modified.'
|
||||
|
||||
def create_migration_file
|
||||
unless file_name =~ /^[_a-z0-9]+$/
|
||||
raise ActiveRecord::IllegalMigrationNameError.new(file_name)
|
||||
end
|
||||
|
||||
migration_template 'migration.rb',
|
||||
migration_template 'migration.rb.erb',
|
||||
"db/migrate/#{file_name}.#{CanvasPartman.migrations_scope}.rb"
|
||||
end
|
||||
|
||||
|
|
@ -25,4 +25,4 @@ class PartitionMigrationGenerator < ActiveRecord::Generators::MigrationGenerator
|
|||
def migration_class_name
|
||||
name.camelize
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -1,10 +1,5 @@
|
|||
class <%= migration_class_name %> < CanvasPartman::Migration
|
||||
self.master_table = '<%= master_table || "name_of_master_table" %>'
|
||||
|
||||
# If the base class can not be infered from the master table name because, for
|
||||
# example, it is namespaced, you may explicitly specify it here:
|
||||
#
|
||||
# self.base_class = Quizzes::QuizSubmissionEvent
|
||||
self.base_class = '<%= model || "NameOfModel" %>'
|
||||
|
||||
def self.up
|
||||
with_each_partition do |partition|
|
||||
|
|
@ -50,4 +50,37 @@ describe CanvasPartman::PartitionManager do
|
|||
end
|
||||
end
|
||||
end
|
||||
|
||||
describe "#ensure_partitions" do
|
||||
it "should create the proper number of partitions" do
|
||||
expect(subject).to receive(:partition_exists?).at_least(:once).and_return(false)
|
||||
expect(Time).to receive(:now).and_return(Time.utc(2015, 05, 02))
|
||||
expect(subject).to receive(:create_partition).with(Time.utc(2015, 05, 01))
|
||||
expect(subject).to receive(:create_partition).with(Time.utc(2015, 06, 01))
|
||||
|
||||
subject.ensure_partitions(1)
|
||||
end
|
||||
end
|
||||
|
||||
describe "#prune_partitions" do
|
||||
it "should prune the proper number of partitions" do
|
||||
expect(Time).to receive(:now).and_return(Time.utc(2015, 05, 02))
|
||||
expect(subject).to receive(:partition_tables).and_return(%w{
|
||||
partman_animals_2014_9
|
||||
partman_animals_2014_10
|
||||
partman_animals_2014_11
|
||||
partman_animals_2014_12
|
||||
partman_animals_2015_1
|
||||
partman_animals_2015_2
|
||||
partman_animals_2015_3
|
||||
partman_animals_2015_4
|
||||
partman_animals_2015_5
|
||||
partman_animals_2015_6
|
||||
})
|
||||
|
||||
expect(subject.base_class.connection).to receive(:drop_table).with('partman_animals_2014_9')
|
||||
expect(subject.base_class.connection).to receive(:drop_table).with('partman_animals_2014_10')
|
||||
subject.prune_partitions(6)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
class AddFooToPartmanAnimals < CanvasPartman::Migration
|
||||
self.master_table = :partman_animals
|
||||
self.base_class = Animal
|
||||
|
||||
def self.up
|
||||
with_each_partition do |partition_table_name|
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
class AddBarToPartmanAnimals < CanvasPartman::Migration
|
||||
self.master_table = :partman_animals
|
||||
self.base_class = Animal
|
||||
|
||||
def change
|
||||
with_each_partition do |table_name|
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
class RemoveFooFromPartmanAnimals < CanvasPartman::Migration
|
||||
self.master_table = :partman_animals
|
||||
self.base_class = Animal
|
||||
|
||||
def self.up
|
||||
with_each_partition do |partition_table_name|
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@ module CanvasPartmanTest
|
|||
end
|
||||
|
||||
class AddAnotherThingToPartmanAnimals < CanvasPartman::Migration
|
||||
self.master_table = 'partman_animals'
|
||||
self.base_class = CanvasPartmanTest::AnimalAlias
|
||||
|
||||
def self.up
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
class AddRaceIndexToPartmanAnimals < CanvasPartman::Migration
|
||||
self.master_table = 'partman_animals'
|
||||
self.base_class = Animal
|
||||
|
||||
def self.up
|
||||
with_each_partition do |partition_table_name|
|
||||
|
|
|
|||
Loading…
Reference in New Issue