ylqjgm
/
canvas-lms
mirror of https://github.com/instructure/canvas-lms.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

add support for tar and tar.gz archives in content migrations 

also add limits for byte size and file count to prevent
zip/tar "bombs"

test plan:
 * import the package referenced in the ticket
 * should import successfully

 * content migration regressions

closes #CNVS-14303 #CNVS-14428

Change-Id: Ia424b5260e34f35b62ca47f7aafa77118c4f5b5b
Reviewed-on: https://gerrit.instructure.com/37881
Reviewed-by: Jeremy Stanley <jeremy@instructure.com>
Tested-by: Jenkins <jenkins@instructure.com>
QA-Review: Trevor deHaan <tdehaan@instructure.com>
Product-Review: James Williams  <jamesw@instructure.com>
This commit is contained in:
James Williams 2014-07-31 07:14:01 -06:00
parent 00b6f88161
commit 829478bc38
23 changed files with 415 additions and 175 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
gems/canvas_unzip/Gemfile
View File

@ -1,2 +1,3 @@
source 'https://rubygems.org'
gem 'canvas_mimetype_fu', :path => '../canvas_mimetype_fu'
gemspec

1
gems/canvas_unzip/canvas_unzip.gemspec
View File

@ -15,6 +15,7 @@ Gem::Specification.new do |spec|
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
spec.require_paths = ["lib"]
spec.add_dependency "canvas_mimetype_fu"
spec.add_dependency "rubyzip", "1.1.1"
spec.add_development_dependency "bundler", "~> 1.5"

186
gems/canvas_unzip/lib/canvas_unzip.rb
View File

@ -17,11 +17,22 @@
require 'zip'
require 'fileutils'
require 'canvas_mimetype_fu'
require 'rubygems/package'
require 'zlib'
class CanvasUnzip
class CanvasUnzipError < ::StandardError; end
class UnknownArchiveType < CanvasUnzipError; end
class FileLimitExceeded < CanvasUnzipError; end
class SizeLimitExceeded < CanvasUnzipError; end
class DestinationFileExists < CanvasUnzipError; end
Limits = Struct.new(:maximum_bytes, :maximum_files)
def self.unsafe_entry?(entry)
return entry.symlink? || entry.name[0] == '/' || entry.name.split('/').include?('..')
entry.symlink? || entry.name == '/' || entry.name.split('/').include?('..')
end
def self.add_warning(warnings, entry, tag)
@ -29,40 +40,167 @@ class CanvasUnzip
warnings[tag] << entry.name
end
BUFFER_SIZE = 65536
DEFAULT_BYTE_LIMIT = 50 << 30
def self.default_limits(file_size)
# * maximum byte count is, unless specified otherwise,
# 100x the size of the uploaded zip, or a hard cap at 50GB
# * default maximum file count is 100,000
Limits.new([file_size * 100, DEFAULT_BYTE_LIMIT].min, 100_000)
end
# if a destination path is given, the archive will be extracted to that location
# * if a block is given, it will be called to ask whether an existing file should be overwritten
# yields |zip_entry, dest_path|; return true to overwrite
# * if no block is given, files will be skipped if they already exist
# if no destination path is specified, then a block must be given
# * yields |zip_entry, index| for each (safe) zip entry
# * files will be skipped if they already exist
# if no destination path is given, a block must be given,
# * yields |entry, index| for each (safe) zip/tar entry available to be extracted
# returns a hash of lists of entries that were skipped by reason
# { :unsafe => [list of entries],
# :already_exists => [list of entries],
# :unknown_compression_method => [list of entries] }
def self.extract_archive(zip_filename, dest_path = nil, &block)
def self.extract_archive(archive_filename, dest_folder = nil, limits = nil, &block)
warnings = {}
Zip::File.open(zip_filename) do |zipfile|
zipfile.entries.each_with_index do |entry, index|
if unsafe_entry?(entry)
add_warning(warnings, entry, :unsafe)
next
end
if dest_path
f_path = File.join(dest_path, entry.name)
FileUtils.mkdir_p(File.dirname(f_path))
begin
entry.extract(f_path, &block)
rescue Zip::DestinationFileExistsError
add_warning(warnings, entry, :already_exists)
rescue Zip::CompressionMethodError
add_warning(warnings, entry, :unknown_compression_method)
limits ||= default_limits(File.size(archive_filename))
bytes_left = limits.maximum_bytes
files_left = limits.maximum_files
raise ArgumentError, "File not found" unless File.exists?(archive_filename)
raise ArgumentError, "Needs block or destination path" unless dest_folder || block
each_entry(archive_filename) do |entry, index|
if unsafe_entry?(entry)
add_warning(warnings, entry, :unsafe)
next
end
if block
block.call(entry, index)
else
raise FileLimitExceeded if files_left <= 0
begin
f_path = File.join(dest_folder, entry.name)
entry.extract(f_path, false, bytes_left) do |size|
bytes_left -= size
raise SizeLimitExceeded if bytes_left < 0
end
else
block.call(entry, index)
files_left -= 1
rescue DestinationFileExists
add_warning(warnings, entry, :already_exists)
rescue Zip::CompressionMethodError
add_warning(warnings, entry, :unknown_compression_method)
end
end
end
warnings
end
def self.each_entry(archive_filename)
raise ArgumentError, "no block given" unless block_given?
file = File.open(archive_filename)
mime_type = File.mime_type?(file)
if mime_type == 'application/x-gzip'
file = Zlib::GzipReader.new(file)
mime_type = 'application/x-tar' # it may not actually be a tar though, so rescue if there's a problem
end
if mime_type == 'application/zip'
Zip::File.open(file) do |zipfile|
zipfile.entries.each_with_index do |zip_entry, index|
yield(Entry.new(zip_entry), index)
end
end
elsif mime_type == 'application/x-tar'
index = 0
begin
Gem::Package::TarReader.new(file).each do |tar_entry|
next if tar_entry.header.typeflag == 'x'
yield(Entry.new(tar_entry), index)
index += 1
end
rescue Gem::Package::TarInvalidError
raise UnknownArchiveType
end
else
raise UnknownArchiveType
end
end
class Entry
attr_reader :entry, :type
def initialize(entry)
if entry.is_a?(Zip::Entry)
@type = :zip
elsif entry.is_a?(Gem::Package::TarReader::Entry)
@type = :tar
end
raise CanvasUnzipError, "Invalid entry type" unless @type
@entry = entry
end
def symlink?
if type == :zip
entry.symlink?
elsif type == :tar
entry.header.typeflag == "2"
end
end
def directory?
entry.directory?
end
def file?
entry.file?
end
def name
if type == :zip
entry.name
elsif type == :tar
entry.full_name.sub(/^\.\//, '')
end
end
def size
if type == :zip
entry.size
elsif type == :tar
entry.header.size
end
end
# yields byte count
def extract(dest_path, overwrite=false, maximum_size=DEFAULT_BYTE_LIMIT)
dir = self.directory? ? dest_path : File.dirname(dest_path)
FileUtils.mkdir_p(dir) unless File.exist?(dir)
return unless self.file?
raise SizeLimitExceeded if size > maximum_size
if File.exist?(dest_path) && !overwrite
raise DestinationFileExists, "Destination '#{dest_path}' already exists"
end
::File.open(dest_path, "wb") do |os|
if type == :zip
entry.get_input_stream do |is|
entry.set_extra_attributes_on_path(dest_path)
buf = ''
while buf = is.sysread(::Zip::Decompressor::CHUNK_SIZE, buf)
os << buf
yield(buf.size) if block_given?
end
end
elsif type == :tar
while buf = entry.read(BUFFER_SIZE)
os << buf
yield(buf.size) if block_given?
end
end
end
end
end
end

119
gems/canvas_unzip/spec/canvas_unzip_spec.rb
View File

@ -26,62 +26,79 @@ def fixture_filename(fixture)
end
describe "CanvasUnzip" do
it "should extract an archive" do
Dir.mktmpdir do |tmpdir|
warnings = CanvasUnzip.extract_archive(fixture_filename('test.zip'), tmpdir)
expect(warnings).to eq({})
expect(File.directory?(File.join(tmpdir, 'empty_dir'))).to be true
expect(File.read(File.join(tmpdir, 'file1.txt'))).to eq "file1\n"
expect(File.read(File.join(tmpdir, 'sub_dir/file2.txt'))).to eq "file2\n"
expect(File.read(File.join(tmpdir, 'implicit_dir/file3.txt'))).to eq "file3\n"
end
end
it "should skip files that already exist by default" do
Dir.mktmpdir do |tmpdir|
File.open(File.join(tmpdir, 'file1.txt'), 'w') { |f| f.puts "OOGA" }
warnings = CanvasUnzip.extract_archive(fixture_filename('test.zip'), tmpdir)
expect(warnings).to eq({already_exists: ['file1.txt']})
expect(File.read(File.join(tmpdir, 'file1.txt'))).to eq "OOGA\n"
expect(File.read(File.join(tmpdir, 'sub_dir/file2.txt'))).to eq "file2\n"
end
end
it "should overwrite files if specified by the block" do
Dir.mktmpdir do |tmpdir|
File.open(File.join(tmpdir, 'file1.txt'), 'w') { |f| f.puts "OOGA" }
Dir.mkdir(File.join(tmpdir, 'sub_dir'))
File.open(File.join(tmpdir, 'sub_dir/file2.txt'), 'w') { |f| f.puts "BOOGA" }
warnings = CanvasUnzip.extract_archive(fixture_filename('test.zip'), tmpdir) do |entry, path|
entry.name == 'sub_dir/file2.txt'
shared_examples_for 'it extracts archives with extension' do |extension|
it "should extract an archive" do
Dir.mktmpdir do |tmpdir|
warnings = CanvasUnzip.extract_archive(fixture_filename("test.#{extension}"), tmpdir)
expect(warnings).to eq({})
expect(File.directory?(File.join(tmpdir, 'empty_dir'))).to be true
expect(File.read(File.join(tmpdir, 'file1.txt'))).to eq "file1\n"
expect(File.read(File.join(tmpdir, 'sub_dir/file2.txt'))).to eq "file2\n"
expect(File.read(File.join(tmpdir, 'implicit_dir/file3.txt'))).to eq "file3\n"
end
expect(warnings).to eq({already_exists: ['file1.txt']})
expect(File.read(File.join(tmpdir, 'file1.txt'))).to eq "OOGA\n"
expect(File.read(File.join(tmpdir, 'sub_dir/file2.txt'))).to eq "file2\n"
end
it "should skip files that already exist by default" do
Dir.mktmpdir do |tmpdir|
File.open(File.join(tmpdir, 'file1.txt'), 'w') { |f| f.puts "OOGA" }
warnings = CanvasUnzip.extract_archive(fixture_filename("test.#{extension}"), tmpdir)
expect(warnings).to eq({already_exists: ['file1.txt']})
expect(File.read(File.join(tmpdir, 'file1.txt'))).to eq "OOGA\n"
expect(File.read(File.join(tmpdir, 'sub_dir/file2.txt'))).to eq "file2\n"
end
end
it "should skip unsafe entries" do
Dir.mktmpdir do |tmpdir|
subdir = File.join(tmpdir, 'sub_dir')
Dir.mkdir(subdir)
warnings = CanvasUnzip.extract_archive(fixture_filename("evil.#{extension}"), subdir)
expect(warnings[:unsafe].sort).to eq ["../outside.txt", "evil_symlink", "tricky/../../outside.txt"]
expect(File.exists?(File.join(tmpdir, 'outside.txt'))).to be false
expect(File.exists?(File.join(subdir, 'evil_symlink'))).to be false
expect(File.exists?(File.join(subdir, 'inside.txt'))).to be true
end
end
it "should enumerate entries" do
indices = []
entries = []
warnings = CanvasUnzip.extract_archive(fixture_filename("evil.#{extension}")) do |entry, index|
entries << entry
indices << index
end
expect(warnings[:unsafe].sort).to eq ["../outside.txt", "evil_symlink", "tricky/../../outside.txt"]
expect(indices.uniq.sort).to eq(indices)
expect(entries.map(&:name)).to eq(['inside.txt', 'tricky/', 'tricky/innocuous_file'])
end
end
it "should skip unsafe entries" do
Dir.mktmpdir do |tmpdir|
subdir = File.join(tmpdir, 'sub_dir')
Dir.mkdir(subdir)
warnings = CanvasUnzip.extract_archive(fixture_filename('evil.zip'), subdir)
expect(warnings).to eq({unsafe: ['evil_symlink', '../outside.txt', 'tricky/../../outside.txt']})
expect(File.exists?(File.join(tmpdir, 'outside.txt'))).to be false
expect(File.exists?(File.join(subdir, 'evil_symlink'))).to be false
expect(File.exists?(File.join(subdir, 'inside.txt'))).to be true
describe "Limits" do
it "should compute reasonable default limits" do
expect(CanvasUnzip.default_limits(100).maximum_bytes).to eq 10_000
expect(CanvasUnzip.default_limits(1_000_000_000).maximum_bytes).to eq CanvasUnzip::DEFAULT_BYTE_LIMIT
end
it "should raise an error if the file limit is exceeded" do
expect {
limits = CanvasUnzip::Limits.new(CanvasUnzip::DEFAULT_BYTE_LIMIT, 2)
Dir.mktmpdir do |tmpdir|
CanvasUnzip.extract_archive(fixture_filename("test.zip"), tmpdir, limits)
end
}.to raise_error(CanvasUnzip::FileLimitExceeded)
end
it "should raise an error if the byte limit is exceeded" do
expect {
limits = CanvasUnzip::Limits.new(10, 100)
Dir.mktmpdir do |tmpdir|
CanvasUnzip.extract_archive(fixture_filename("test.zip"), tmpdir, limits)
end
}.to raise_error(CanvasUnzip::SizeLimitExceeded)
end
end
it "should enumerate entries" do
indices = []
entries = []
warnings = CanvasUnzip.extract_archive(fixture_filename('evil.zip')) do |entry, index|
entries << entry
indices << index
end
expect(warnings).to eq({unsafe: ['evil_symlink', '../outside.txt', 'tricky/../../outside.txt']})
expect(indices.uniq.sort).to eq(indices)
expect(entries.map(&:name)).to eq(['inside.txt', 'tricky/', 'tricky/innocuous_file'])
end
it_behaves_like 'it extracts archives with extension', 'zip'
it_behaves_like 'it extracts archives with extension', 'tar'
it_behaves_like 'it extracts archives with extension', 'tar.gz'
end

BIN
gems/canvas_unzip/spec/fixtures/evil.tar vendored Normal file
View File

Binary file not shown.

BIN
gems/canvas_unzip/spec/fixtures/evil.tar.gz vendored Normal file
View File

Binary file not shown.

BIN
gems/canvas_unzip/spec/fixtures/test.tar vendored Normal file
View File

Binary file not shown.

BIN
gems/canvas_unzip/spec/fixtures/test.tar.gz vendored Normal file
View File

Binary file not shown.

106
lib/canvas/migration/archive.rb Normal file
View File

@ -0,0 +1,106 @@
module Canvas::Migration
class Archive
attr_reader :warnings
def initialize(settings={})
@settings = settings
@warnings = []
end
def file
@file ||= @settings[:archive_file] || download_archive
end
def zip_file
@zip_file ||= Zip::File.open(file) rescue false
end
def read(entry)
if zip_file
zip_file.read(entry)
else
unzip_archive
path = File.join(self.unzipped_file_path, entry)
File.exists?(path) && File.read(path)
end
end
def find_entry(entry)
if zip_file
zip_file.find_entry(entry)
else
# if it's not an actual zip file
# just extract the package (or try to) and look for the file
unzip_archive
File.exists?(File.join(self.unzipped_file_path, entry))
end
end
def download_archive
config = ConfigFile.load('external_migration') || {}
if @settings[:export_archive_path]
File.open(@settings[:export_archive_path], 'rb')
elsif @settings[:course_archive_download_url].present?
# open-uri downloads the http response to a tempfile
open(@settings[:course_archive_download_url])
elsif @settings[:attachment_id]
att = Attachment.find(@settings[:attachment_id])
att.open(:temp_folder => config[:data_folder], :need_local_file => true)
else
raise "No migration file found"
end
end
def path
file.path
end
def unzipped_file_path
unless @unzipped_file_path
config = ConfigFile.load('external_migration') || {}
@unzipped_file_path = Dir.mktmpdir(nil, config[:data_folder].presence)
end
@unzipped_file_path
end
def get_converter
Canvas::Migration::PackageIdentifier.new(self).get_converter
end
def unzip_archive
return if @unzipped
Rails.logger.debug "Extracting #{path} to #{unzipped_file_path}"
warnings = CanvasUnzip.extract_archive(path, unzipped_file_path)
@unzipped = true
unless warnings.empty?
diagnostic_text = ''
warnings.each do |tag, files|
diagnostic_text += tag.to_s + ': ' + files.join(', ') + "\n"
end
Rails.logger.debug "CanvasUnzip returned warnings: " + diagnostic_text
add_warning(I18n.t('canvas.migration.warning.unzip_warning', 'The content package unzipped successfully, but with a warning'), diagnostic_text)
end
return true
end
# If the file is a zip file, unzip it, if it's an xml file, copy
# it into the directory with the given file name
def prepare_cartridge_file(file_name='imsmanifest.xml')
if self.path.ends_with?('xml')
FileUtils::cp(self.path, File.join(self.unzipped_file_path, file_name))
else
unzip_archive
end
end
def delete_unzipped_file
if File.exists?(self.unzipped_file_path)
FileUtils::rm_rf(self.unzipped_file_path)
end
end
def add_warning(warning)
@warnings << warning
end
end
end

53
lib/canvas/migration/migrator.rb
View File

@ -34,17 +34,13 @@ class Migrator
@course = {:file_map=>{}, :wikis=>[]}
@course[:name] = @settings[:course_name]
unless settings[:no_archive_file]
unless settings[:archive_file]
MigratorHelper::download_archive(settings)
end
if @archive_file = settings[:archive_file]
@archive_file_path = @archive_file.path
end
unless @settings[:no_archive_file]
@archive = @settings[:archive] || Canvas::Migration::Archive.new(@settings)
@archive_file = @archive.file
@unzipped_file_path = @archive.unzipped_file_path
@archive_file_path = @archive.path
end
config = ConfigFile.load('external_migration') || {}
@unzipped_file_path = Dir.mktmpdir(migration_type.to_s, config[:data_folder].presence)
@base_export_dir = @settings[:base_download_dir] || find_export_dir
@course[:export_folder_path] = File.expand_path(@base_export_dir)
make_export_dir
@ -55,40 +51,17 @@ class Migrator
end
def unzip_archive
logger.debug "Extracting #{@archive_file_path} to #{@unzipped_file_path}"
warnings = CanvasUnzip.extract_archive(@archive_file_path, @unzipped_file_path)
unless warnings.empty?
diagnostic_text = ''
warnings.each do |tag, files|
diagnostic_text += tag.to_s + ': ' + files.join(', ') + "\n"
end
logger.debug "CanvasUnzip returned warnings: " + diagnostic_text
add_warning(I18n.t('canvas.migration.warning.unzip_warning', 'The content package unzipped successfully, but with a warning'), diagnostic_text)
end
return true
end
# If the file is a zip file, unzip it, if it's an xml file, copy
# it into the directory with the given file name
def prepare_cartridge_file(file_name='imsmanifest.xml')
if @archive_file_path.ends_with?('xml')
FileUtils::cp(@archive_file_path, File.join(@unzipped_file_path, file_name))
else
unzip_archive
@archive.unzip_archive
@archive.warnings.each do |warn|
add_warning(warn)
end
end
def delete_unzipped_archive
delete_file(@unzipped_file_path)
end
def delete_file(file)
if File.exists?(file)
begin
FileUtils::rm_rf(file)
rescue
Rails.logger.warn "Couldn't delete #{file} for content_migration #{@settings[:content_migration_id]}"
end
begin
@archive.delete_unzipped_archive
rescue
Rails.logger.warn "Couldn't delete #{@unzipped_file_path} for content_migration #{@settings[:content_migration_id]}"
end
end

17
lib/canvas/migration/migrator_helper.rb
View File

@ -148,23 +148,6 @@ module MigratorHelper
File.open(file_name, 'w') { |file| file << @course.to_json}
file_name
end
def self.download_archive(settings)
config = ConfigFile.load('external_migration') || {}
if settings[:export_archive_path]
settings[:archive_file] = File.open(settings[:export_archive_path], 'rb')
elsif settings[:course_archive_download_url].present?
# open-uri downloads the http response to a tempfile
settings[:archive_file] = open(settings[:course_archive_download_url])
elsif settings[:attachment_id]
att = Attachment.find(settings[:attachment_id])
settings[:archive_file] = att.open(:temp_folder => config[:data_folder], :need_local_file => true)
else
raise "No migration file found"
end
settings[:archive_file]
end
def id_prepender
@settings[:id_prepender]

29
lib/canvas/migration/package_identifier.rb
View File

@ -3,12 +3,8 @@ module Canvas::Migration
include Canvas::Migration::XMLHelper
attr_reader :type, :converter
def initialize(settings)
unless settings[:archive_file]
MigratorHelper::download_archive(settings)
end
@archive = settings[:archive_file]
@type = :unknown
def initialize(archive)
@archive = archive
end
def get_converter
@ -21,17 +17,16 @@ module Canvas::Migration
return check_flat_xml_file
end
zip_file = Zip::File.open(@archive.path)
if zip_file.find_entry("AngelManifest.xml")
if @archive.find_entry("AngelManifest.xml")
:angel_7_4
elsif zip_file.find_entry("angelData.xml")
elsif @archive.find_entry("angelData.xml")
:angel_7_3
elsif zip_file.find_entry("moodle.xml")
elsif @archive.find_entry("moodle.xml")
:moodle_1_9
elsif zip_file.find_entry("moodle_backup.xml")
elsif @archive.find_entry("moodle_backup.xml")
:moodle_2
elsif zip_file.find_entry("imsmanifest.xml")
data = zip_file.read("imsmanifest.xml")
elsif @archive.find_entry("imsmanifest.xml")
data = @archive.read("imsmanifest.xml")
doc = ::Nokogiri::XML(data)
if get_node_val(doc, 'metadata schema') =~ /IMS Common Cartridge/i
if !!doc.at_css(%{resources resource[href="#{CC::CCHelper::COURSE_SETTINGS_DIR}/#{CC::CCHelper::SYLLABUS}"] file[href="#{CC::CCHelper::COURSE_SETTINGS_DIR}/#{CC::CCHelper::COURSE_SETTINGS}"]})
@ -72,8 +67,8 @@ module Canvas::Migration
else
:unknown
end
rescue Zip::Error
# Not a valid zip file
rescue
# Not a valid archive file
:invalid_archive
end
@ -82,7 +77,7 @@ module Canvas::Migration
# Common Cartridge 1.3 supports having just a single xml file
# if it's not CC 1.3 then we don't know how to handle it
def check_flat_xml_file
doc = ::Nokogiri::XML(File.read(@archive))
doc = ::Nokogiri::XML(File.read(@archive.file))
if get_node_val(doc, 'metadata schema') =~ /IMS Common Cartridge/i &&
get_node_val(doc, 'metadata schemaversion') == "1.3.0"
:common_cartridge_1_3
@ -90,7 +85,6 @@ module Canvas::Migration
:unknown
end
end
def has_namespace(node, namespace)
node.namespaces.values.any?{|ns|ns =~ /#{namespace}/i}
@ -100,6 +94,7 @@ module Canvas::Migration
if plugin = Canvas::Plugin.all_for_tag(:export_system).find{|p|p.settings[:provides] && p.settings[:provides][@type]}
return plugin.settings[:provides][@type]
end
raise Canvas::Migration::Error, I18n.t(:unsupported_package, "Unsupported content package")
end
end

4
lib/canvas/migration/worker.rb
View File

@ -21,9 +21,9 @@ require 'action_controller_test_process'
module Canvas::Migration::Worker
def self.get_converter(settings)
Canvas::Migration::PackageIdentifier.new(settings).get_converter
Canvas::Migration::Archive.new(settings).get_converter
end
def self.upload_overview_file(file, content_migration)
uploaded_data = Rack::Test::UploadedFile.new(file.path, Attachment.mimetype(file.path))

9
lib/cc/importer/cc_worker.rb
View File

@ -36,7 +36,14 @@ class Canvas::Migration::Worker::CCWorker < Struct.new(:migration_id)
raise Canvas::Migration::Error, I18n.t(:no_migration_file, "File required for content migration.")
end
converter_class = settings[:converter_class] || Canvas::Migration::Worker::get_converter(settings)
converter_class = settings[:converter_class]
unless converter_class
if settings[:no_archive_file]
raise ArgumentError, "converter_class required for content migration with no file"
end
settings[:archive] = Canvas::Migration::Archive.new(settings)
converter_class = settings[:archive].get_converter
end
converter = converter_class.new(settings)
course = converter.export

2
lib/cc/importer/standard/converter.rb
View File

@ -41,7 +41,7 @@ module CC::Importer::Standard
# exports the package into the intermediary json
def convert(to_export = nil)
prepare_cartridge_file(MANIFEST_FILE)
@archive.prepare_cartridge_file(MANIFEST_FILE)
@manifest = open_file_xml(File.join(@unzipped_file_path, MANIFEST_FILE))
@manifest.remove_namespaces!

17
lib/unzip_attachment.rb
View File

@ -113,7 +113,9 @@ class UnzipAttachment
# have to worry about what this name actually is.
Tempfile.open(filename) do |f|
begin
extract_entry(entry, f.path)
entry.extract(f.path, true) do |bytes|
zip_stats.charge_quota(bytes)
end
# This is where the attachment actually happens. See file_in_context.rb
attachment = attach(f.path, entry, folder)
id_positions[attachment.id] = path_positions[entry.name]
@ -138,19 +140,6 @@ class UnzipAttachment
update_progress(1.0)
end
def extract_entry(entry, dest_path)
::File.open(dest_path, "wb") do |os|
entry.get_input_stream do |is|
entry.set_extra_attributes_on_path(dest_path)
buf = ''
while buf = is.sysread(::Zip::Decompressor::CHUNK_SIZE, buf)
os << buf
zip_stats.charge_quota(buf.size)
end
end
end
end
def zip_stats
@zip_stats ||= ZipFileStats.new(filename)
end

BIN
spec/fixtures/migration/cc_default_qb_test.tar.gz vendored Normal file
View File

Binary file not shown.

3
spec/lib/canvas/migration_spec.rb
View File

@ -50,7 +50,8 @@ describe "Migration package importers" do
unsupported.each_pair do |key, val|
it "should correctly identify package type for #{key}" do
settings = get_settings(val.first)
Canvas::Migration::PackageIdentifier.new(settings).identify_package.should == val.last
archive = Canvas::Migration::Archive.new(settings)
Canvas::Migration::PackageIdentifier.new(archive).identify_package.should == val.last
end
end
end

8
spec/lib/cc/importer/cc_worker_spec.rb
View File

@ -20,9 +20,9 @@ require File.expand_path(File.dirname(__FILE__) + '/../../../spec_helper.rb')
describe Canvas::Migration::Worker::CCWorker do
it "should set the worker_class on the migration" do
cm = ContentMigration.create!(:migration_settings => { :no_archive_file => true }, :context => course)
cm = ContentMigration.create!(:migration_settings => { :converter_class => CC::Importer::Canvas::Converter,
:no_archive_file => true }, :context => course)
cm.reset_job_progress
Canvas::Migration::Worker.expects(:get_converter).with(anything).returns(CC::Importer::Canvas::Converter)
CC::Importer::Canvas::Converter.any_instance.expects(:export).returns({})
worker = Canvas::Migration::Worker::CCWorker.new(cm.id)
worker.perform().should == true
@ -30,8 +30,8 @@ describe Canvas::Migration::Worker::CCWorker do
end
it "should honor skip_job_progress" do
cm = ContentMigration.create!(:migration_settings => { :no_archive_file => true, :skip_job_progress => true }, :context => course)
Canvas::Migration::Worker.expects(:get_converter).with(anything).returns(CC::Importer::Canvas::Converter)
cm = ContentMigration.create!(:migration_settings => { :converter_class => CC::Importer::Canvas::Converter,
:no_archive_file => true, :skip_job_progress => true }, :context => course)
CC::Importer::Canvas::Converter.any_instance.expects(:export).returns({})
worker = Canvas::Migration::Worker::CCWorker.new(cm.id)
worker.perform().should == true

27
spec/models/content_migration_spec.rb
View File

@ -2697,4 +2697,31 @@ equation: <img class="equation_image" title="Log_216" src="/equation_images/Log_
bank.assessment_questions.count.should == 1
end
end
it "should identify and import compressed tarball archives" do
pending unless Qti.qti_enabled?
course_with_teacher
cm = ContentMigration.new(:context => @course, :user => @user)
cm.migration_type = 'qti_converter'
cm.migration_settings['import_immediately'] = true
cm.save!
package_path = File.join(File.dirname(__FILE__) + "/../fixtures/migration/cc_default_qb_test.tar.gz")
attachment = Attachment.new
attachment.context = cm
attachment.uploaded_data = File.open(package_path, 'rb')
attachment.filename = 'file.zip'
attachment.save!
cm.attachment = attachment
cm.save!
cm.queue_migration
run_jobs
cm.migration_issues.should be_empty
@course.assessment_question_banks.count.should == 1
end
end

2
vendor/plugins/moodle2cc/Gemfile vendored
View File

@ -1,3 +1,3 @@
gem 'moodle2cc', '0.2.12'
gem 'moodle2cc', '0.2.14'
gem 'happymapper', '0.4.1'
gem 'thor', '0.18.1'

5
vendor/plugins/moodle2cc/lib/moodle/converter.rb vendored
View File

@ -5,7 +5,8 @@ module Moodle
end
def export(to_export = Canvas::Migration::Migrator::SCRAPE_ALL_HASH)
migrator = Moodle2CC::Migrator.new @archive_file.path, @unzipped_file_path, 'format' => 'canvas', 'logger' => self
unzip_archive
migrator = Moodle2CC::Migrator.new @unzipped_file_path, Dir.mktmpdir, 'format' => 'canvas', 'logger' => self
migrator.migrate
if migrator.last_error
@ -13,6 +14,8 @@ module Moodle
end
@settings[:archive_file] = File.open(migrator.imscc_path)
@settings.delete(:archive)
cc_converter = CC::Importer::Canvas::Converter.new(@settings)
cc_converter.export
@course = cc_converter.course

1
vendor/plugins/moodle2cc/spec_canvas/moodle2_converter_spec.rb vendored
View File

@ -31,7 +31,6 @@ describe Moodle::Converter do
it "should convert discussion topics" do
@course.discussion_topics.count.should == 2
pending("moodle2cc 0.2.14")
dt = @course.discussion_topics.first
dt.title.should == "Hidden Forum"
dt.message.should == "<p>Description of hidden forum</p>"