ylqjgm
/
canvas-lms
mirror of https://github.com/instructure/canvas-lms.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

yank migrations and table management from canvas ddb 

Change-Id: I3d2a60443e9b514f690a236b05dd2f437ad8e3c2
Reviewed-on: https://gerrit.instructure.com/c/canvas-lms/+/246391
Reviewed-by: Michael Ziwisky <mziwisky@instructure.com>
Tested-by: Service Cloud Jenkins <svc.cloudjenkins@instructure.com>
QA-Review: Ethan Vizitei <evizitei@instructure.com>
Product-Review: Ethan Vizitei <evizitei@instructure.com>
This commit is contained in:
Ethan Vizitei 2020-08-28 15:42:04 -05:00
parent 391a31b978
commit 16abf15de4
12 changed files with 203 additions and 241 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
app/graphql/audit_log_field_extension.rb
View File

@ -31,7 +31,7 @@ class AuditLogFieldExtension < GraphQL::Schema::FieldExtension
def log(entry, field_name)
@dynamo.put_item(
table_name: "graphql_mutations",
table_name: AuditLogFieldExtension.ddb_table_name,
item: {
# TODO: this is where you redirect
"object_id" => log_entry_id(entry, field_name),
@ -125,6 +125,10 @@ class AuditLogFieldExtension < GraphQL::Schema::FieldExtension
Canvas::DynamoDB::DatabaseBuilder.configured?(:auditors)
end
def self.ddb_table_name
Setting.get("graphql_mutations_ddb_table_name", "graphql_mutations")
end
def resolve(object:, arguments:, context:, **rest)
yield(object, arguments).tap do |value|
next unless AuditLogFieldExtension.enabled?

2
app/graphql/types/audit_logs_type.rb
View File

@ -33,7 +33,7 @@ module Types
start_time ||= 1.year.ago
end_time ||= 1.year.from_now
DynamoQuery.new(dynamo, "graphql_mutations",
DynamoQuery.new(dynamo, AuditLogFieldExtension.ddb_table_name,
partition_key: "object_id",
value: "#{context[:domain_root_account].global_id}-#{asset_string}",
key_condition_expression: "mutation_id BETWEEN :start_time AND :end_time",

42
db/migrate/20190415173518_create_mutation_audit_log.rb
View File

@ -1,42 +0,0 @@
#
# Copyright (C) 2019 - present Instructure, Inc.
#
# This file is part of Canvas.
#
# Canvas is free software: you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License as published by the Free
# Software Foundation, version 3 of the License.
#
# Canvas is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
# details.
#
# You should have received a copy of the GNU Affero General Public License along
# with this program. If not, see <http://www.gnu.org/licenses/>.
#
class CreateMutationAuditLog < ActiveRecord::Migration[5.1]
tag :predeploy
include Canvas::DynamoDB::Migration
category :auditors
def self.up
create_table table_name: :graphql_mutations,
ttl_attribute: "expires",
attribute_definitions: [
{attribute_name: "object_id", attribute_type: "S"},
{attribute_name: "mutation_id", attribute_type: "S"},
],
key_schema: [
{attribute_name: "object_id", key_type: "HASH"},
{attribute_name: "mutation_id", key_type: "RANGE"},
]
end
def self.down
delete_table table_name: :graphql_mutations
end
end

15
gems/canvas_dynamodb/README.md Normal file
View File

@ -0,0 +1,15 @@
# Canvas DynamoDB
An opinionated way to talk to Dynamo tables.
## Features
- *Query Logging* : statements that get executed will get written to rails debug logs
- *Batch Operations*: Useful helper methods for building batched read/write queries.
- *Table Naming Conventions*: using prefixes and semantic names.
## Intentional Omissions!
We don't want to manage the tables themselves. A DynamoDB table is
more like a database than a schema change. It should be
managed in terraform and scaled by devops tooling.

104
gems/canvas_dynamodb/lib/canvas_dynamodb/database.rb
View File

@ -22,19 +22,17 @@ module CanvasDynamoDB
DEFAULT_MIN_CAPACITY = 5
DEFAULT_MAX_CAPACITY = 10000
attr_reader :client, :fingerprint
attr_reader :client, :fingerprint, :logger
def initialize(fingerprint, prefix, autoscaling_role_arn, opts, logger)
def initialize(fingerprint, prefix, opts, logger_arg)
@client = Aws::DynamoDB::Client.new(opts)
@region = opts[:region]
@fingerprint = fingerprint
@prefix = prefix
@autoscaling_role_arn = autoscaling_role_arn
@logger = logger
@logger = logger_arg
end
%i(create_table delete_item delete_table get_item put_item query scan update_item
update_table update_time_to_live describe_time_to_live).each do |method|
%i(delete_item get_item put_item query scan update_item).each do |method|
define_method(method) do |params|
params = params.merge(
table_name: prefixed_table_name(params[:table_name])
@ -43,15 +41,6 @@ module CanvasDynamoDB
end
end
%i(create_global_table update_global_table).each do |method|
define_method(method) do |params|
params = params.merge(
global_table_name: prefixed_table_name(params[:global_table_name])
)
execute(method, params)
end
end
%i(batch_get_item batch_write_item).each do |method|
define_method(method) do |params|
request_items = {}
@ -77,92 +66,11 @@ module CanvasDynamoDB
def execute(method, params)
result = nil
ms = 1000 * Benchmark.realtime do
result = @client.send(method, params)
result = client.send(method, params)
end
@logger.debug(" #{"DDB (%.2fms)" % [ms]} #{method}(#{params.inspect}) [#{fingerprint}]")
logger.debug(" #{"DDB (%.2fms)" % [ms]} #{method}(#{params.inspect}) [#{fingerprint}]")
result
end
def create_table_with_autoscaling(params)
out = create_table(params)
if scaling
scaling.register_scalable_target(register_scaling_target_params(
params[:table_name],
:read,
min_capacity: params.dig(:provisioned_throughput, :read_capacity_units)
))
scaling.put_scaling_policy(scaling_policy_params(params[:table_name], :read))
scaling.register_scalable_target(register_scaling_target_params(
params[:table_name],
:write,
min_capacity: params.dig(:provisioned_throughput, :write_capacity_units)
))
scaling.put_scaling_policy(scaling_policy_params(params[:table_name], :write))
end
out
end
def delete_table_with_autoscaling(params)
if scaling
scaling.deregister_scalable_target(scaling_target_params(params[:table_name], :read,))
scaling.deregister_scalable_target(scaling_target_params(params[:table_name], :write))
end
delete_table(params)
end
def scaling
@scaling ||= begin
if @autoscaling_role_arn
require 'aws-sdk-applicationautoscaling'
Aws::ApplicationAutoScaling::Client.new({ region: @region })
end
end
end
private
def scaling_target_params(table_name, rw)
scalable_dimension = rw == :read ?
'dynamodb:table:ReadCapacityUnits' :
'dynamodb:table:WriteCapacityUnits'
{
resource_id: "table/#{@prefix}#{table_name}",
scalable_dimension: scalable_dimension,
service_namespace: "dynamodb",
}
end
def register_scaling_target_params(table_name, rw, min_capacity: nil, max_capacity: nil)
scaling_target_params(table_name, rw).merge({
min_capacity: min_capacity || DEFAULT_MIN_CAPACITY,
max_capacity: max_capacity || DEFAULT_MAX_CAPACITY,
role_arn: @autoscaling_role_arn,
})
end
def scaling_policy_params(table_name, rw)
predefined_metric_type = rw == :read ?
'DynamoDBReadCapacityUtilization' :
'DynamoDBWriteCapacityUtilization'
scalable_dimension = rw == :read ?
'dynamodb:table:ReadCapacityUnits' :
'dynamodb:table:WriteCapacityUnits'
{
resource_id: "table/#{@prefix}#{table_name}",
policy_name: "#{@prefix}#{table_name}--#{predefined_metric_type}",
policy_type: 'TargetTrackingScaling',
scalable_dimension: scalable_dimension,
service_namespace: "dynamodb",
target_tracking_scaling_policy_configuration: {
target_value: 70.0,
predefined_metric_specification: {
predefined_metric_type: predefined_metric_type
},
scale_out_cooldown: 60,
scale_in_cooldown: 60
}
}
end
end
end

65
gems/canvas_dynamodb/spec/lib/database_spec.rb Normal file
View File

@ -0,0 +1,65 @@
#
# Copyright (C) 2014 - present Instructure, Inc.
#
# This file is part of Canvas.
#
# Canvas is free software: you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License as published by the Free
# Software Foundation, version 3 of the License.
#
# Canvas is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
# details.
#
# You should have received a copy of the GNU Affero General Public License along
# with this program. If not, see <http://www.gnu.org/licenses/>.
#
require "spec_helper"
describe CanvasDynamoDB::Database do
let(:db) do
fingerprint = "asdf"
prefix = "sdfa"
logger_cls = Class.new do
attr_reader :messages
def initialize
@messages = []
end
def debug(msg)
@messages << msg
end
end
CanvasDynamoDB::Database.new(fingerprint, prefix, {}, logger_cls.new)
end
describe "#get_item" do
before(:each) do
ddb_client_class = Class.new do
attr_reader :last_query
def get_item(hash)
@last_query = hash
{}
end
end
allow(Aws::DynamoDB::Client).to receive(:new).and_return(ddb_client_class.new)
end
it "prefixes table name on query" do
db.get_item(table_name: "my_table", key: {object_id: "dfas", mutation_id: "fasd"})
expect(db.client.last_query[:table_name]).to eq("sdfa-my_table")
end
it "logs requests" do
db.get_item(table_name: "my_table", key: {object_id: "adsf", mutation_id: "afsd"})
log_message = db.logger.messages.last
expect(log_message).to match(/ DDB /)
expect(log_message).to match(/\d+.\d+ms/)
expect(log_message).to include("get_item({:table_name=>\"sdfa-my_table\"")
end
end
end

35
gems/canvas_dynamodb/spec/spec_helper.rb Normal file
View File

@ -0,0 +1,35 @@
#
# Copyright (C) 2020 - present Instructure, Inc.
#
# This file is part of Canvas.
#
# Canvas is free software: you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License as published by the Free
# Software Foundation, version 3 of the License.
#
# Canvas is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
# details.
#
# You should have received a copy of the GNU Affero General Public License along
# with this program. If not, see <http://www.gnu.org/licenses/>.
#
require "canvas_dynamodb"
require "yaml"
RSpec.configure do |config|
config.run_all_when_everything_filtered = true
config.filter_run :focus
# Run specs in random order to surface order dependencies. If you find an
# order dependency and want to debug it, you can fix the order by providing
# the seed, which is printed after each run.
# --seed 1234
config.order = 'random'
config.expect_with :rspec do |c|
c.syntax = :expect
end
end

4
lib/canvas/dynamo_db/database_builder.rb
View File

@ -47,7 +47,6 @@ module Canvas
@clients[key] = CanvasDynamoDB::Database.new(
fingerprint,
config[:table_prefix],
config[:autoscaling_role_arn],
opts,
Rails.logger
)
@ -79,9 +78,6 @@ module Canvas
configs.keys
end
def self.read_consistency_setting(category)
Cavas::Cassandra::DatabaseBuilder.read_consistency_setting(category)
end
end
end
end

71
lib/canvas/dynamo_db/dev_utils.rb Normal file
View File

@ -0,0 +1,71 @@
#
# Copyright (C) 2020 - present Instructure, Inc.
#
# This file is part of Canvas.
#
# Canvas is free software: you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License as published by the Free
# Software Foundation, version 3 of the License.
#
# Canvas is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
# details.
#
# You should have received a copy of the GNU Affero General Public License along
# with this program. If not, see <http://www.gnu.org/licenses/>.
module Canvas
module DynamoDB
module DevUtils
SCHEMA_FIXTURES = {
'graphql_mutations' => {
attribute_definitions: [
{attribute_name: "object_id", attribute_type: "S"},
{attribute_name: "mutation_id", attribute_type: "S"},
],
key_schema: [
{attribute_name: "object_id", key_type: "HASH"},
{attribute_name: "mutation_id", key_type: "RANGE"},
]
}
}.freeze
def self.initialize_ddb_for_development!(category, table_name, recreate: false, schema: nil)
unless ["development", "test"].include?(Rails.env)
raise "DynamoDB should not be initialized this way in a real environment!!!"
end
canvas_ddb = Canvas::DynamoDB::DatabaseBuilder.from_config(category)
dynamodb = canvas_ddb.client
local_table_name = canvas_ddb.prefixed_table_name(table_name)
exists = begin
result = dynamodb.describe_table(table_name: local_table_name)
true
rescue Aws::DynamoDB::Errors::ResourceNotFoundException
false
end
if exists
Rails.logger.debug("Local DDB table #{local_table_name} already exists!")
return true unless recreate
Rails.logger.debug("Deleting existing table...")
dynamodb.delete_table(table_name: local_table_name)
end
Rails.logger.debug("Creating local DDB table for #{local_table_name}...")
schema_opts = schema || SCHEMA_FIXTURES[table_name]
params = schema_opts.merge({
table_name: local_table_name,
provisioned_throughput: { read_capacity_units: 5, write_capacity_units: 5 }
})
begin
result = dynamodb.create_table(params)
Rails.logger.debug('Created table. Status: ' + result.table_description.table_status)
return true
rescue Aws::DynamoDB::Errors::ServiceError => error
Rails.logger.debug('Unable to create table:')
Rails.logger.debug(error.message)
return false
end
end
end
end
end

94
lib/canvas/dynamo_db/migration.rb
View File

@ -1,94 +0,0 @@
#
# Copyright (C) 2018 - present Instructure, Inc.
#
# This file is part of Canvas.
#
# Canvas is free software: you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License as published by the Free
# Software Foundation, version 3 of the License.
#
# Canvas is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
# details.
#
# You should have received a copy of the GNU Affero General Public License along
# with this program. If not, see <http://www.gnu.org/licenses/>.
module Canvas
module DynamoDB
module Migration
DEFAULT_READ_CAPACITY_UNITS = 5
DEFAULT_WRITE_CAPACITY_UNITS = 5
module ClassMethods
def db
@dyanmodb ||= Canvas::DynamoDB::DatabaseBuilder.from_config(category)
end
def category(category = nil)
return @dynamodb_category if category.nil?
@dynamodb_category = category
end
def runnable?
raise "Configuration category is required to be set" unless category.present?
Switchman::Shard.current == Switchman::Shard.birth &&
Canvas::DynamoDB::DatabaseBuilder.configured?(category)
end
def create_table(params)
ttl_attribute = params.delete(:ttl_attribute)
params = provisioned_throughput.merge(params)
params[:global_secondary_indexes].try(:each_with_index) do |gsi, idx|
params[:global_secondary_indexes][idx] = provisioned_throughput.merge(gsi)
end
db.create_table_with_autoscaling(params)
if ttl_attribute
db.update_time_to_live({
table_name: params[:table_name],
time_to_live_specification: {
enabled: true,
attribute_name: ttl_attribute,
},
})
end
end
def delete_table(params)
db.delete_table_with_autoscaling(params)
end
private
def read_capacity_units
key = "dynamodb_migration_read_capacity_units"
units = Setting.get("#{key}_#{category}", nil) if category
units ||= Setting.get(key, DEFAULT_READ_CAPACITY_UNITS )
end
def write_capacity_units
key = "dynamodb_migration_write_capacity_units"
units = Setting.get("#{key}_#{category}", nil) if category
units ||= Setting.get(key, DEFAULT_READ_CAPACITY_UNITS)
end
def provisioned_throughput
{
provisioned_throughput: {
read_capacity_units: read_capacity_units,
write_capacity_units: write_capacity_units,
}
}
end
end
def self.included(migration)
migration.tag :dynamodb
migration.singleton_class.include(ClassMethods)
end
end
end
end

5
spec/graphql/mutation_audit_log_spec.rb
View File

@ -20,6 +20,7 @@ require_relative "../spec_helper"
require_relative "./graphql_spec_helper"
describe AuditLogFieldExtension do
before do
if !AuditLogFieldExtension.enabled?
skip("AuditLog needs to be enabled by configuring dynamodb.yml")
@ -27,6 +28,7 @@ describe AuditLogFieldExtension do
end
before(:once) do
Canvas::DynamoDB::DevUtils.initialize_ddb_for_development!(:auditors, "graphql_mutations", recreate: true)
course_with_student(active_all: true)
@assignment = @course.assignments.create! name: "asdf"
MUTATION = <<~MUTATION
@ -62,7 +64,7 @@ describe AuditLogFieldExtension do
it "fails gracefully when dynamo isn't working, with captured exception" do
require 'canvas_dynamodb'
dynamo = CanvasDynamoDB::Database.new("asdf", "asdf", nil,
dynamo = CanvasDynamoDB::Database.new("asdf", "asdf",
{region: "us-east-1", endpoint: "http://localhost:8000"},
Rails.logger)
expect(dynamo).to receive(:put_item).and_raise(Aws::DynamoDB::Errors::ServiceError.new("two", "arguments"))
@ -81,6 +83,7 @@ describe AuditLogFieldExtension::Logger do
let(:mutation) { double(graphql_name: "asdf") }
before(:once) do
Canvas::DynamoDB::DevUtils.initialize_ddb_for_development!(:auditors, "graphql_mutations", recreate: true)
course_with_teacher(active_all: true)
@entry = @course.assignments.create! name: "asdf"
end

1
spec/graphql/types/mutation_log_type_spec.rb
View File

@ -27,6 +27,7 @@ describe Types::MutationLogType do
end
before(:once) do
Canvas::DynamoDB::DevUtils.initialize_ddb_for_development!(:auditors, "graphql_mutations", recreate: true)
student_in_course(active_all: true)
@assignment = @course.assignments.create! name: "asdf"
account_admin_user