canvas-lms/lib/content_zipper.rb

#
# Copyright (C) 2011 Instructure, Inc.
#
# This file is part of Canvas.
#
# Canvas is free software: you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License as published by the Free
# Software Foundation, version 3 of the License.
#
# Canvas is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
# details.
#
# You should have received a copy of the GNU Affero General Public License along
# with this program. If not, see <http://www.gnu.org/licenses/>.
#
require 'zip/zip'
require 'action_controller'
require 'action_controller/test_process.rb'
require 'tmpdir'
require 'set'

class ContentZipper

  def initialize(options={})
    @check_user = options.has_key?(:check_user) ? options[:check_user] : true
    @logger = Rails.logger
  end
  
  def self.send_later_if_production(*args)
    if ENV['RAILS_ENV'] == 'production'
      send_later(*args)
    else
      send(*args)
    end
  end
  
  def send_later_if_production(*args)
    if ENV['RAILS_ENV'] == 'production'
      send_later(*args)
    else
      send(*args)
    end
  end

  # we evaluate some ERB templates from under app/views/ while generating assignment zips
  include I18nUtilities
  def t(*a, &b)
    I18n.t(*a, &b)
  end

  def self.process_attachment(*args)
    options = args.last.is_a?(Hash) ? args.pop : {} 
    ContentZipper.new(options).process_attachment(*args)
  end
  
  def process_attachment(attachment, user = nil)
    raise "No attachment provided to ContentZipper.process_attachment" unless attachment
    
    attachment.update_attribute(:workflow_state, 'zipping')
    @user = user
    @logger.debug("file found: #{attachment.id} zipping files...")
    
    begin
      case attachment.context
      when Assignment; zip_assignment(attachment, attachment.context)
      when Eportfolio; zip_eportfolio(attachment, attachment.context)
      when Folder; zip_base_folder(attachment, attachment.context)
      end
    rescue => e
      ErrorReport.log_exception(:default, e, {
        :message => "Content zipping failed",
      })
      @logger.debug(e.to_s)
      @logger.debug(e.backtrace.join('\n'))
      attachment.update_attribute(:workflow_state, 'to_be_zipped')
    end
  end
  
  def assignment_zip_filename(assignment)
    "#{assignment.context.short_name_slug}-#{assignment.title_slug} submissions"
  end

  def zip_assignment(zip_attachment, assignment)
    files = []
    @logger.debug("zipping into attachment: #{zip_attachment.id}")
    zip_attachment.workflow_state = 'zipping'
    zip_attachment.scribd_attempts += 1
    zip_attachment.save!
    filename = assignment_zip_filename(assignment)
    submissions = assignment.submissions
    if zip_attachment.user && assignment.context.enrollment_visibility_level_for(zip_attachment.user) != :full
      visible_student_ids = assignment.context.enrollments_visible_to(zip_attachment.user).find(:all, :select => 'user_id').map(&:user_id)
      submissions = submissions.scoped(:conditions => { :user_id => visible_student_ids})
    end
    make_zip_tmpdir(filename) do |zip_name|
      @logger.debug("creating #{zip_name}")
      submissions_added = 0
      Zip::ZipFile.open(zip_name, Zip::ZipFile::CREATE) do |zipfile|
        count = submissions.length
        submissions.each_with_index do |submission, idx|
          submissions_added += 1
          @assignment = assignment
          @submission = submission
          @context = assignment.context
          @logger.debug(" checking submission for #{(submission.user.name rescue nil)}")

          # it's necessary to replace _\d+_ because we use that pattern to infer the user/attachment ids when teachers
          # upload graded submissions
          users_name = submission.user.last_name_first.gsub(/_(\d+)_/, '-\1-')

          filename = users_name + (submission.late? ? " LATE " : " ") + submission.user_id.to_s
          filename = filename.gsub(/ /, "_").gsub(/[^-\w]/, "").downcase
          content = nil
          if submission.submission_type == "online_upload"
            submission.attachments.each do |attachment|
              @logger.debug("  found attachment: #{attachment.display_name}")
              fn = filename + "_" + attachment.id.to_s + "_" + attachment.display_name
              if add_attachment_to_zip(attachment, zipfile, fn)
                files << fn
              end
            end
          elsif submission.submission_type == "online_url" && submission.url
            @logger.debug("  found url: #{submission.url}")
            self.extend(ApplicationHelper)
            filename += "_link.html"
            @logger.debug("  loading template")
            content = File.open(File.join("app", "views", "assignments", "redirect_page.html.erb")).read
            @logger.debug("  parsing template")
            content = ERB.new(content).result(binding)
            @logger.debug("  done parsing template")
            if content
              zipfile.get_output_stream(filename) {|f| f.puts content }
              files << filename
            end
          elsif submission.submission_type == "online_text_entry" && submission.body
            @logger.debug("  found text entry")
            self.extend(ApplicationHelper)
            filename += "_text.html"
            content = File.open(File.join("app", "views", "assignments", "text_entry_page.html.erb")).read
            content = ERB.new(content).result(binding)
            if content
              zipfile.get_output_stream(filename) {|f| f.puts content }
              files << filename
            end
          end
          zip_attachment.workflow_state = 'zipping'
          zip_attachment.file_state = ((idx + 1).to_f / count.to_f * 100).to_i
          zip_attachment.save!
          @logger.debug("status for #{zip_attachment.id} updated to #{zip_attachment.file_state}")
        end
      end
      @logger.debug("added #{submissions_added} submissions")
      assignment.increment!(:submissions_downloads)
      if files.empty?
        zip_attachment.workflow_state = 'errored'
        zip_attachment.save!
      else
        @logger.debug("data zipped! uploading to s3...")
        uploaded_data = ActionController::TestUploadedFile.new(zip_name, 'application/zip')
        zip_attachment.uploaded_data = uploaded_data
        zip_attachment.workflow_state = 'zipped'
        zip_attachment.file_state = 'available'
        zip_attachment.save!
      end
    end
  end
  
  def self.zip_eportfolio(*args)
    ContentZipper.new.zip_eportfolio(*args)
  end

  StaticAttachment = Struct.new(:display_name,
                                :filename,
                                :content_type,
                                :uuid,
                                :attachment)
  
  def zip_eportfolio(zip_attachment, portfolio)
    static_attachments = []
    submissions = []
    portfolio.eportfolio_entries.each do |entry|
      static_attachments += entry.attachments
      submissions += entry.submissions
    end
    idx = 1
    submissions_hash = {}
    submissions.each do |s|
      submissions_hash[s.id] = s
      if s.submission_type == 'online_upload'
        static_attachments += s.attachments
      else
      end
    end
    static_attachments = static_attachments.uniq.map do |a|
      obj = StaticAttachment.new
      obj.display_name = a.display_name
      obj.filename = "#{idx}_#{a.filename}"
      obj.content_type = a.content_type
      obj.uuid = a.uuid
      obj.attachment = a
      idx += 1
      obj
    end
    filename = portfolio.name
    make_zip_tmpdir(filename) do |zip_name|
      idx = 0
      count = static_attachments.length + 2
      Zip::ZipFile.open(zip_name, Zip::ZipFile::CREATE) do |zipfile|
        zip_attachment.file_state = ((idx + 1).to_f / count.to_f * 100).to_i
        zip_attachment.save!
        portfolio.eportfolio_entries.each do |entry|
          filename = "#{entry.full_slug}.html"
          content = render_eportfolio_page_content(entry, portfolio, static_attachments, submissions_hash)
          zipfile.get_output_stream(filename) {|f| f.puts content }
        end
        zip_attachment.file_state = ((idx + 1).to_f / count.to_f * 100).to_i
        zip_attachment.save!
        static_attachments.each do |a|
          add_attachment_to_zip(a.attachment, zipfile)
          zip_attachment.file_state = ((idx + 1).to_f / count.to_f * 100).to_i
          zip_attachment.save!
        end
        if css = File.open(File.join(RAILS_ROOT, 'public', 'stylesheets', 'static', 'eportfolio_static.css')) rescue nil
          content = css.read
          zipfile.get_output_stream("eportfolio.css") {|f| f.puts content } if content
        end
        content = File.open(File.join(RAILS_ROOT, 'public', 'images', 'logo.png'), 'rb').read rescue nil
        zipfile.get_output_stream("logo.png") {|f| f.write content } if content
      end
      @logger.debug("data zipped!")
      uploaded_data = ActionController::TestUploadedFile.new(zip_name, 'application/zip')
      zip_attachment.uploaded_data = uploaded_data
      zip_attachment.workflow_state = 'zipped'
      zip_attachment.file_state = 'available'
      zip_attachment.save!
    end
  end

  def render_eportfolio_page_content(page, portfolio, static_attachments, submissions_hash)
    @page = page
    @portfolio = @portfolio
    @static_attachments = static_attachments
    @submissions_hash = submissions_hash
    av = ActionView::Base.new(Rails::Configuration.new.view_path)
    av.extend TextHelper
    res = av.render(:partial => "eportfolios/static_page", :locals => {:page => page, :portfolio => portfolio, :static_attachments => static_attachments, :submissions_hash => submissions_hash})
    res
  end
  
  def self.zip_base_folder(*args)
    ContentZipper.new.zip_base_folder(*args)
  end
  
  def zip_base_folder(zip_attachment, folder)
    @files_added = true
    @logger.debug("zipping into attachment: #{zip_attachment.id}")
    zip_attachment.workflow_state = 'zipping' #!(:workflow_state => 'zipping')
    zip_attachment.scribd_attempts += 1
    zip_attachment.save!
    filename = "#{folder.context.short_name}-#{folder.name} files"
    make_zip_tmpdir(filename) do |zip_name|
      @logger.debug("creating #{zip_name}")
      Zip::ZipFile.open(zip_name, Zip::ZipFile::CREATE) do |zipfile|
        @logger.debug("zip_name: #{zip_name}")
        process_folder(folder, zipfile)
      end
      if @files_added
        @logger.debug("data zipped!")
        uploaded_data = ActionController::TestUploadedFile.new(zip_name, 'application/zip')
        zip_attachment.uploaded_data = uploaded_data
        zip_attachment.workflow_state = 'zipped'
        zip_attachment.file_state = 'available'
        zip_attachment.save!
      else
        zip_attachment.workflow_state = 'errored'
        zip_attachment.save!
      end
    end
  end
  
  def process_folder(folder, zipfile, start_dirs=[], &callback)
    if callback
      zip_folder(folder, zipfile, start_dirs, &callback)
    else
      zip_folder(folder, zipfile, start_dirs)
    end
  end
  
  # make a tmp directory and yield a filename under that directory to the block
  # given. the tmp directory is deleted when the block returns.
  def make_zip_tmpdir(filename)
    filename = File.basename(filename.gsub(/ /, "_").gsub(/[^\w-]/, ""))
    Dir.mktmpdir do |dirname|
      zip_name = File.join(dirname, "#{filename}.zip")
      yield zip_name
    end
  end
  
  # The callback should accept two arguments, the attachment/folder and the folder names
  def zip_folder(folder, zipfile, folder_names, &callback)
    if callback && (folder.hidden? || folder.locked)
      callback.call(folder, folder_names)
    end
    # @user = nil either means that 
    # 1. this is part of a public course, and is being downloaded by somebody
    # not logged in - OR -
    # 2. we're doing this inside a course context export, and are bypassing
    # the user check (@check_user == false)
    attachments = if !@check_user || folder.context.grants_right?(@user, nil, :manage_files)
                folder.active_file_attachments
              else
                folder.visible_file_attachments
              end
    attachments.select{|a| !@check_user || a.grants_right?(@user, nil, :download)}.each do |attachment|
      callback.call(attachment, folder_names) if callback
      @context = folder.context
      @logger.debug("  found attachment: #{attachment.unencoded_filename}")
      path = folder_names.empty? ? attachment.display_name : File.join(folder_names, attachment.display_name)
      @files_added = false unless add_attachment_to_zip(attachment, zipfile, path)
    end
    folder.active_sub_folders.select{|f| !@check_user || f.grants_right?(@user, nil, :read_contents)}.each do |sub_folder|
      new_names = Array.new(folder_names) << sub_folder.name
      if callback
        zip_folder(sub_folder, zipfile, new_names, &callback)
      else
        zip_folder(sub_folder, zipfile, new_names)
      end
    end
  end
  
  def add_attachment_to_zip(attachment, zipfile, filename = nil)
    filename ||= attachment.filename

    # we allow duplicate filenames in the same folder. it's a bit silly, but we
    # have to handle it here or people might not get all their files zipped up.
    @files_in_zip ||= Set.new
    filename = Attachment.make_unique_filename(filename, @files_in_zip)
    @files_in_zip << filename
    
    handle = nil
    begin
      handle = attachment.open(:need_local_file => true)
      zipfile.get_output_stream(filename){|zos| IOExtras.copy_stream(zos, handle)}
    rescue => e
      @logger.error("  skipping #{attachment.full_filename} with error: #{e.message}")
      return false
    ensure
      handle.close if handle
    end
    
    true
  end
end