ylqjgm
/
canvas-lms
mirror of https://github.com/instructure/canvas-lms.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove ModelGenerator class and rake task 

This is in a separate repository (model-generator) now.

Change-Id: I5974799efe41d44b624dc037991d684e1c64a26e
Reviewed-on: https://gerrit.instructure.com/c/canvas-lms/+/237022
Tested-by: Service Cloud Jenkins <svc.cloudjenkins@instructure.com>
Product-Review: Jeremy Slade <jslade@instructure.com>
Reviewed-by: Jeremy Slade <jslade@instructure.com>
QA-Review: Tucker Mcknight <tmcknight@instructure.com>
This commit is contained in:
Tucker McKnight 2020-05-12 00:17:06 -06:00 committed by Tucker Mcknight
parent b67d8f02ab
commit 4f1cde8964
3 changed files with 0 additions and 569 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

261
lib/cdc_migration_testing/model_generator.rb
View File

@ -1,261 +0,0 @@
#
# Copyright (C) 2020 - present Instructure, Inc.
#
# This file is part of Canvas.
#
# Canvas is free software: you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License as published by the Free
# Software Foundation, version 3 of the License.
#
# Canvas is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
# details.
#
# You should have received a copy of the GNU Affero General Public License along
# with this program. If not, see <http://www.gnu.org/licenses/>.
#
class ModelGenerator
FIXTURES_BASEDIR = 'lib/cdc_migration_testing/fixtures'.freeze
attr_reader :generated_count, :fixture_count, :skipped_count, :iteration_count
def initialize
Rails.application.eager_load!
@model_queue = ActiveRecord::Base.descendants
@generated_count = @fixture_count = @skipped_count = @iteration_count = 0
end
def queue_length
@model_queue.length
end
def run
loop do
last_queue_length = @model_queue.length
create_models
@iteration_count+=1
break if @model_queue.empty?
# If no models were created in that run (so the queue length didn't change),
# something is preventing it from advancing.
raise "Couldn't generate all necessary models." if last_queue_length == @model_queue.length
end
end
private
def create_models
@model_queue.each do
model = @model_queue.shift()
if !model.table_exists? || records?(model.table_name_prefix + model.table_name)
@skipped_count+=1
next
end
begin
create_model(model)
# If there's a foreign key error, put this model back at the end of the
# queue and try again later so that the prerequisite models can
# be created first. This takes several cycles, but finishes eventually.
rescue ActiveRecord::InvalidForeignKey
@model_queue.push(model)
rescue StandardError => e
raise <<~ERROR
Couldn't create a #{model.name}. If one can't be generated, you
will need to define a CdcFixtures.create_#{model.class_name.underscore} method in
#{fixture_file_path(model)} that returns a #{model.name}.\n Error message:\n #{e.message}
ERROR
end
end
end
def create_model(model)
remove_callbacks(model)
begin
create_from_fixture_file(model)
# TypeError is thrown if the file doesn't exist.
rescue TypeError
Rails.logger.info "No sample #{model.name} found in #{fixture_file_path(model)}, will try to auto-generate one."
generate_model(model)
end
end
def remove_callbacks(model)
callbacks = model.__callbacks
[:create, :update, :save, :commit].each do |callback_type|
callbacks[callback_type].each do |callback|
model.skip_callback(callback.name, callback.kind, callback.filter)
end
end
end
def create_from_fixture_file(model)
model_file_path = fixture_file_path(model)
begin
require Rails.root.join(model_file_path)
created_model = CdcFixtures.send("create_#{model.class_name.underscore}".to_sym)
created_model.save!(validate: false)
@fixture_count+=1
rescue ActiveRecord::RecordInvalid => e
raise <<~NOSAVE
Couldn't save the #{model.name} returned by the self.create method in
#{model_file_path}. Make sure that the self.create method returns a
valid #{model.name}.\n
Exception:\n
#{e.message}
NOSAVE
rescue NoMethodError => e
raise <<~ERR
There was an unknown error when loading #{model_file_path} and calling
CdcFixtures.create_#{model.class_name.underscore}.\n
Exception:\n
#{e.message}
ERR
end
end
def generate_model(model)
required_attributes = get_postgres_non_nullable_attributes(model)
required_attributes.merge! get_partman_attributes(model)
required_attributes.merge! set_inheritance_column(model)
begin
model.new(required_attributes).save!(validate: false)
@generated_count+=1
rescue ActiveRecord::ReadOnlyRecord
@skipped_count +=1
# If we just tried to create a read-only record, ignore it.
# AssignmentStudentVisibility, (and possibly other models?) don't
# allow creating records directly for some reason. These are not
# important tables and can be safely ignored.
end
end
# Partman is a gem in gems/canvas_partman that tries to do some automatic
# relationships based on column names like "thing_id" and "thing_type".
# It is not avoided by skipping validations or callbacks, so we have to fill
# in those _id columns.
def get_partman_attributes(model)
relation_names = find_partman_columns(model)
attributes = {}
relation_names.each do |relation_name|
attributes[relation_name + '_id'] = 1
attributes[relation_name + '_type'] = polymorphic_class_name(model, relation_name)
end
attributes
end
def find_partman_columns(model)
model.columns.map { |column|
is_partman_column = false
relation_name = nil
# Look for a column name that ends with _id, then look for a matching
# column that ends with _type.
if column.name.end_with?("_id")
column_without_suffix = column.name[0..-4]
is_partman_column = model.columns.any?{ |col| col.name == column_without_suffix + '_type' }
relation_name = column_without_suffix if is_partman_column
end
relation_name
}.compact
end
# This method tries to find a valid class name to fill in for columns that store
# the class name in a polymorphic relationship. (E.g., the 'context_type' column.)
# Uses "Account" as a default value.
def polymorphic_class_name(model, relation_name)
class_name = 'Account'
reflections = model.reflections[relation_name].options[:polymorphic] rescue nil
reflections&.each do |reflection|
# These key/value pairs look like { underscored_name: 'ClassName' }.
# A value is only given if underscored_name does not describe a class.
# If a value is there, that is the class name.
if reflection.is_a? Hash
class_name = reflection.values.first
break
end
# If a value was not given, turn underscored_name into a class name.
classified_name = ActiveSupport::Inflector.classify(reflection)
# Check if the class name exists.
if ActiveSupport::Inflector.constantize(classified_name)
class_name = classified_name
break
end
end
class_name
end
def get_postgres_non_nullable_attributes(model)
required_columns = model.columns.select do |column|
# column.default_function means that postgres will auto-fill something
# on INSERT (e.g., an auto-incremented ID). So we don't have to fill in
# those columns, unless it's the 'id' field. Then we want to hard-code
# that to 1, since other models always use 1 for foreign key fields.
!column.null && (!column.default_function || column.name == 'id')
end
attributes = {}
required_columns.each do |column|
case column.type
when :integer
attributes[column.name] = 1
when :decimal, :float
attributes[column.name] = 0.9
when :string, :text
attributes[column.name] = string_or_serializable_object(model, column)
when :boolean
attributes[column.name] = false
when :datetime
attributes[column.name] = Time.zone.now
when :json, :jsonb
attributes[column.name] = {foo: 'bar'}
else
raise "Model #{model.name} has no fixture and no default value for column '#{column.name}' of type '#{column.type}'"
end
end
attributes
end
def string_or_serializable_object(model, column)
# A "coder" is defined if the column is a string or text type, but is going
# to be encoded/decoded into a Hash, Array, etc. by ActiveRecord.
if defined? model.attribute_types[column.name].coder
class_name = model.attribute_types[column.name].coder.object_class.name
case class_name
when 'Hash', 'Object'
{foo: 'bar'}
when 'Array'
['foo']
else raise "Model #{model.name} has serializable column #{column.name} of unknown type #{class_name}"
end
else
char_limit = 8
char_limit = [column.limit, char_limit].min if column.limit
'a' * char_limit
end
end
def fixture_file_path(model)
FIXTURES_BASEDIR + '/' + model.name.underscore + '.rb'
end
def set_inheritance_column(model)
attrs = {}
attrs = {'type' => model.descendants.first.name} if model.has_attribute?(:type)
attrs
end
def records?(table_name)
ActiveRecord::Base.connection.exec_query("SELECT * FROM #{Shard.current.name}.#{table_name}").any?
end
end

14
lib/tasks/canvas.rake
View File

@ -208,20 +208,6 @@ namespace :db do
::ActiveRecord::Base.descendants.each(&:reset_column_information)
Rake::Task['db:migrate'].invoke
end
desc "Auto-generate models in all tables for triggering CDC events"
task :fill_tables => [:environment] do
raise "Run with RAILS_ENV=test" unless Rails.env.test?
require "#{Rails.root}/lib/cdc_migration_testing/model_generator"
model_generator = ModelGenerator.new
puts "Attempting to create #{model_generator.queue_length} models"
model_generator.run
puts "Results:\n"
puts "#{model_generator.fixture_count} models created from fixtures"
puts "#{model_generator.generated_count} models generated automatically"
puts "#{model_generator.skipped_count} models skipped (already existed or have no table)"
puts "Took #{model_generator.iteration_count} iterations to satisfy foreign key constraints"
end
end
end

294
spec/lib/cdc_migration_testing/model_generator_spec.rb
View File

@ -1,294 +0,0 @@
#
# Copyright (C) 2020 - present Instructure, Inc.
#
# This file is part of Canvas.
#
# Canvas is free software: you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License as published by the Free
# Software Foundation, version 3 of the License.
#
# Canvas is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
# details.
#
# You should have received a copy of the GNU Affero General Public License along
# with this program. If not, see <http://www.gnu.org/licenses/>.
#
require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper.rb')
require_dependency 'cdc_migration_testing/model_generator'
# TODO: resolve issues with these tests polluting AR classes, then un-skip
xdescribe ModelGenerator do
let(:generator) { ModelGenerator.new }
before :each do
# Unload CdcFixtures. Tests below expect to reload fixtures with different
# contents, so we clean them up for each test.
hide_const('CdcFixtures')
allow(ActiveRecord::Base).to receive(:descendants).and_return(models)
end
# Make sure that it iterates through the model list until all foreign key
# constraints are satisfied. Here, Account is passed in last, but needs
# to be created first, since most models rely on an account.
context 'with an out-of-order model list' do
let(:models) { [ User, EnrollmentTerm, Course, AuthenticationProvider, Account] }
it 'should populate models when run', custom_timeout: 45.seconds do
models.each { |model| expect(model.any?).to be false }
generator.run
models.each { |model| expect(model.any?).to be true }
end
end
context 'with fixture files' do
let(:models) { [ User, Csp::Domain, Account ] }
before :each do
# ModelGenerator dynamically `require`s files based on model name. Remove
# previously required files so they can be used repeatedly in tests
$LOADED_FEATURES.reject! { |path| path =~ /\/lib\/cdc_migration_testing\/fixtures/ }
stub_const('ModelGenerator::FIXTURES_BASEDIR', 'spec/lib/cdc_migration_testing/fixtures')
end
it 'uses the fixture file for a model' do
generator.run
expect(User.last.name).to eq('CDC Sample User')
end
it 'finds fixture files for classes in modules' do
generator.run
expect(Csp::Domain.last.domain).to eq('example.com')
end
end
context 'with an invalid fixture file' do
let(:models) { [ User ] }
before :each do
$LOADED_FEATURES.reject! { |path| path =~ /\/lib\/cdc_migration_testing\/bad_fixtures/ }
stub_const('ModelGenerator::FIXTURES_BASEDIR', 'spec/lib/cdc_migration_testing/bad_fixtures')
end
it 'throws a useful error when the method name is incorrect' do
expect {
# User fixture file has a bad method name. Error should
# tell you that the method name should be create_user.
generator.run
}.to raise_error(/create_user/)
end
end
context 'with a model' do
def run_and_ignore_exceptions
# ActiveRecord will throw many errors because SampleModel isn't real.
# We really just want to check what the attributes are when it attempts to
# create an object -- we can ignore the exceptions when it fails to save.
begin generator.run rescue StandardError end
end
class SampleModel < ActiveRecord::Base
validates_presence_of :attr_required_in_rails
self.table_name = :users # specifying an existing table gets around some AR exceptions
end
let(:models) { [ SampleModel ] }
before :each do
allow(SampleModel).to receive(:columns).and_return(columns)
end
context 'having multiple column types' do
let(:columns) {[
double("int column", name: 'id', type: :integer, null: false, default_function: false),
double("optional int column", name: 'age', type: :integer, null: true, default_function: false),
double("date column with default", name: 'join_date', type: :datetime, null: false, default_function: 'NOW()')
]}
it 'ignores column that are nullable or have default values' do
expect(SampleModel).not_to receive(:new).with hash_including('age')
expect(SampleModel).not_to receive(:new).with hash_including('join_date')
run_and_ignore_exceptions
end
it 'fills in an integer' do
expect(SampleModel).to receive(:new).with('id' => 1)
run_and_ignore_exceptions
end
context 'with models that share a table' do
# SampleModel uses the users table.
it 'only creates one model per table' do
User.create!
expect(SampleModel).not_to receive(:new)
end
end
end
context 'having string columns' do
let(:columns) {[
double("string column", name: 'name', type: :string, null: false, default_function: false, limit: 255),
double("text column", name: 'description', type: :text, null: false, default_function: false, limit: nil),
double("array column", name: 'array_col', type: :text, null: false, default_function: false, limit: nil),
double("hash column", name: 'hash_col', type: :text, null: false, default_function: false, limit: nil),
double("object column", name: 'obj_col', type: :string, null: false, default_function: false, limit: 255),
double("short string", name: 'middle_initial', type: :string, null: false, default_function: false, limit: 1)
]}
it 'fills in strings' do
expect(SampleModel).to receive(:new).with hash_including({
'name' => 'aaaaaaaa',
'description' => 'aaaaaaaa'
})
run_and_ignore_exceptions
end
it 'respects the character limit' do
expect(SampleModel).to receive(:new).with hash_including('middle_initial' => 'a')
run_and_ignore_exceptions
end
context 'with serialized attributes' do
before :each do
array_coder = double('array coder', object_class: Array)
array_column = double('array column', type: :text, coder: array_coder)
hash_coder = double('hash coder', object_class: Hash)
hash_column = double('hash column', type: :text, coder: hash_coder)
object_coder = double('object coder', object_class: Object)
object_column = double('object column', type: :string, coder: object_coder)
allow(SampleModel).to receive(:attribute_types).and_return({
'array_col' => array_column,
'hash_col' => hash_column,
'obj_col' => object_column
})
end
it 'fills in a hash' do
expect(SampleModel).to receive(:new).with hash_including('hash_col' => instance_of(Hash))
run_and_ignore_exceptions
end
it 'fills in an array' do
expect(SampleModel).to receive(:new).with hash_including('array_col' => instance_of(Array))
run_and_ignore_exceptions
end
it 'fills in an Object' do
expect(SampleModel).to receive(:new).with hash_including('obj_col' => instance_of(Hash))
run_and_ignore_exceptions
end
end
end
context 'having a boolean column' do
let(:columns) {[
double("boolean column", name: 'active', type: :boolean, null: false, default_function: false)
]}
it 'fills in a boolean' do
expect(SampleModel).to receive(:new).with hash_including('active' => false)
run_and_ignore_exceptions
end
end
context 'having a date column' do
let(:columns) {[
double("date column", name: 'birthdate', type: :datetime, null: false, default_function: false)
]}
it 'fills in a date' do
expect(SampleModel).to receive(:new).with hash_including('birthdate' => instance_of(ActiveSupport::TimeWithZone))
run_and_ignore_exceptions
end
end
context 'having json columns' do
let(:columns) {[
double("json column", name: 'json', type: :json, null: false, default_function: false),
double("jsonb column", name: 'jsonb', type: :json, null: false, default_function: false)
]}
it 'fills in a hash' do
expect(SampleModel)
.to receive(:new)
.with hash_including('json' => {foo: 'bar'}, 'jsonb' => {foo: 'bar'})
run_and_ignore_exceptions
end
end
context 'that inherits another model' do
class SampleModelExtender < SampleModel
end
let(:models) { [ SampleModel ] }
let(:columns) {[
double('inheritance column', name: 'type', type: :string, null: false, default_function: false, limit: 255)
]}
it 'fills in the type column' do
allow(SampleModel).to receive(:has_attribute?).with(:type).and_return(true)
expect(SampleModel).to receive(:new).with hash_including('type' => 'SampleModelExtender')
# ActiveRecord::Base.descendants was being overridden from earlier, causing a problem.
allow(SampleModel).to receive(:descendants).and_call_original
run_and_ignore_exceptions
end
end
context 'that has a polymorphic association' do
class AssociatedWithSampleModel < ActiveRecord::Base
belongs_to :context, polymorphic: [:sample_model, :user]
belongs_to :named_relationship, polymorphic: [named: 'Course']
self.table_name = 'users'
end
let(:columns) {[]} # Don't care about SampleModel's columns for this one.
let(:models) { [ AssociatedWithSampleModel ] }
it 'finds an underscored class name' do
allow(AssociatedWithSampleModel).to receive(:columns).and_return([
double('id', name: 'context_id', type: :integer, null: :false, default_function: false),
double('type', name: 'context_type', type: :string, null: false, default_function: false, limit: 255),
double('named id', name: 'named_relationship_id', type: :integer, null: :false, default_function: false),
double('named type', name: 'named_relationship_type', type: :string, null: false, default_function: false, limit: 255)
])
expect(AssociatedWithSampleModel)
.to receive(:new)
.with hash_including({
'context_id' => 1,
'context_type' => 'SampleModel',
'named_relationship_id' => 1,
'named_relationship_type' => 'Course'
})
run_and_ignore_exceptions
end
end
end
context 'with a tableless model' do
class TablelessModel < Tableless
end
let(:models) { [ TablelessModel ] }
it 'does not throw an error' do
expect {
generator.run
}.not_to raise_error
end
end
end