refactor docker_dev_update with improvements

closes: DE-499

Improve the flow of docker_dev_update, incorporate new
rebase_canvas_and_plugins script.

Test Plan:
- Jenkins Passes
- Run docker_dev_update with no uncommitted changes
  -no command options; script updates docker containers
  -with --update-code; rebases repos, updates docker
  -try a variety of skip-plugins and skip-canvas
    -script skips repos as expected
-Run with uncommitted changes to canvas or a plugin
  -no command options; updates docker containers
  -with update-code; prompted to stash or skip
    -rebases correctly if stashed
  -use different combinations of skip-canvas and
   skip-plugins; script will skip as expected

Change-Id: I6cf96f2455ca798bc37d8b324190b955ab8bdaa1
Reviewed-on: https://gerrit.instructure.com/c/canvas-lms/+/259133
Tested-by: Service Cloud Jenkins <svc.cloudjenkins@instructure.com>
Reviewed-by: Andrea Cirulli <andrea.cirulli@instructure.com>
QA-Review: Andrea Cirulli <andrea.cirulli@instructure.com>
Product-Review: James Butters <jbutters@instructure.com>

build/common_docker_build_steps.sh

@@ -61,7 +61,7 @@ function check_gemfile {
   if [[ -e Gemfile.lock ]]; then
     message \
 'For historical reasons, the Canvas Gemfile.lock is not tracked by git. We may
-need to remove it before we can install gems, to prevent conflicting depencency
+need to remove it before we can install gems, to prevent conflicting dependency
 errors.'
     confirm_command 'rm -f Gemfile.lock' || true
   fi

script/canvas_update

@@ -26,8 +26,7 @@ function update_canvas {
   intro_message "Canvas Update"
 
   if [[ -z "$SKIP_CODE" ]] ; then
-    update_plugins
-    rebase_canvas
+    ./script/rebase_canvas_and_plugins.sh
   fi
 
   if [[ -z "$SKIP_DEPS" ]] ; then

script/common.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # This file contains commonly used BASH functions for scripting in canvas-lms,
-# particularly script/canvas_update and script/prepare/prepare . As such,
+# particularly script/canvas_update and script/rebase_canvas_and_plugins . As such,
 # *be careful* when you modify these functions as doing so will impact multiple
 # scripts that likely aren't used or tested in continuous integration builds.
 
@@ -59,46 +59,15 @@ function intro_message {
   echo "-----------------------------" >>"$LOG"
 }
 
-function update_plugin {
-  (
-    cd "$1"
-    if is_git_dir; then
-      echo_console_and_log "  Updating plugin $1 ..."
-      git pull --rebase origin master >>"$LOG" 2>&1
-    fi
-  )
-}
-
-function update_plugins {
-  # Loop through each plugin dir, and if it's a git repo, update it
-  # This needs to be done first so that db:migrate can pull in any plugin-
-  # precipitated changes to the database.
-  for dir in {gems,vendor}; do
-    if [ -d "$dir/plugins" ]; then
-      for plugin in $dir/plugins/*; do update_plugin "$plugin"; done
-    fi
-  done
-}
-
-function checkout_master_canvas {
-  echo_console_and_log "  Checking out canvas-lms master ..."
-  git checkout master >>"$LOG" 2>&1
-}
-
-function rebase_canvas {
-  echo_console_and_log "  Rebasing canvas-lms on HEAD ..."
-  git pull --rebase origin master >>"$LOG" 2>&1
-}
-
 function bundle_install {
   echo_console_and_log "  Installing gems (bundle install) ..."
   rm -f Gemfile.lock* >/dev/null 2>&1
-  bundle install >>"$LOG" 2>&1
+  run_command bundle install >>"$LOG" 2>&1
 }
 
 function bundle_install_with_check {
   echo_console_and_log "  Checking your gems (bundle check) ..."
-  if bundle check >>"$LOG" 2>&1 ; then
+  if run_command bundle check >>"$LOG" 2>&1 ; then
     echo_console_and_log "  Gems are up to date, no need to bundle install ..."
   else
     bundle_install
@@ -107,20 +76,28 @@ function bundle_install_with_check {
 
 function rake_db_migrate_dev_and_test {
   echo_console_and_log "  Migrating development DB ..."
-  RAILS_ENV=development bundle exec rake db:migrate >>"$LOG" 2>&1
-
+  run_command bundle exec rake db:migrate RAILS_ENV=development >>"$LOG" 2>&1
   echo_console_and_log "  Migrating test DB ..."
-  RAILS_ENV=test bundle exec rake db:migrate >>"$LOG" 2>&1
+  run_command bundle exec rake db:migrate RAILS_ENV=test >>"$LOG" 2>&1
 }
 
 function install_node_packages {
   echo_console_and_log "  Installing Node packages ..."
-  bundle exec rake js:yarn_install >>"$LOG" 2>&1
+  run_command bundle exec rake js:yarn_install >>"$LOG" 2>&1
 }
 
 function compile_assets {
   echo_console_and_log "  Compiling assets (css and js only, no docs or styleguide) ..."
-  bundle exec rake canvas:compile_assets_dev >>"$LOG" 2>&1
+  run_command bundle exec rake canvas:compile_assets_dev >>"$LOG" 2>&1
+}
+
+# If DOCKER var set true, run with docker-compose
+function run_command {
+  if [ "${DOCKER:-}" == 'y' ]; then
+    docker-compose run --rm web "$@"
+  else
+    "$@"
+  fi
 }
 
 function _canvas_lms_track {

script/docker_dev_setup.sh

@@ -1,7 +1,6 @@
 #!/bin/bash
 
 set -e
-
 source build/common_docker_build_steps.sh
 
 trap '_canvas_lms_telemetry_report_status' ERR EXIT
@@ -60,6 +59,7 @@ DINGHY_CPUS='4'
 DINGHY_DISK='150'
 # docker-compose version 1.20.0 introduced build-arg that we use for linux
 DOCKER_COMPOSE_MIN_VERSION='1.20.0'
+DOCKER='y'
 
 if [[ $OS == 'Darwin' ]]; then
   #docker-compose is checked separately

script/docker_dev_update.sh

@@ -1,55 +1,93 @@
-#!/bin/bash -e
+#!/bin/bash
+
+set -e
+source script/common.sh
+source build/common_docker_build_steps.sh
+
+LOG="$(pwd)/log/docker_dev_update.log"
+DOCKER='y'
+
+trap print_results EXIT
+trap "printf '\nTerminated\n' && exit 130" SIGINT
 
 usage () {
-  echo "usage: $0 [-f] [-h] [-n phase]"
+  echo "usage:"
+  printf "  --update-code [skip-canvas] [skip-plugins [<plugin1>,...]\tRebase canvas-lms and plugins. Optional skip-canvas and\n"
+  printf " \t\t\t\t\t\t\t\tskip-plugins. Comma separated list of plugins to skip.\n"
+  printf "  -h|--help\t\t\t\t\t\t\tDisplay usage\n\n"
 }
 
-bad_usage () {
+die () {
+  echo "$*" 1>&2
   usage
   exit 1
 }
 
-while getopts ":fhn:" opt
-do
-  case $opt in
-    n )
-      case $OPTARG in
-        build )
-          SKIP_BUILD=true;;
-        code )
-          SKIP_CODE=true;;
-        * )
-          bad_usage;;
-      esac
-      echo "Skipping $OPTARG";;
-    f )
-      FORCE=yes;;
-    h )
-      usage;;
-    * )
-      echo "Sorry, -$OPTARG is not a valid option!"
-      bad_usage;;
+while :; do
+  case $1 in
+    -h|-\?|--help)
+      usage
+      exit
+      ;;
+    --update-code)
+      UPDATE_CODE=true
+      params=()
+      while :; do
+        case $2 in
+          skip-canvas)
+            params+=(--skip-canvas)
+            ;;
+          skip-plugins)
+            if [ "$3" ] && [[ "$3" != "skip-canvas" ]]; then
+              repos=$3
+              params+=(--skip-plugins $repos)
+              shift
+            else
+              params+=(--skip-plugins)
+            fi
+            ;;
+          *)
+            break
+        esac
+        shift
+      done
+      ;;
+    ?*)
+      die 'ERROR: Unknown option: ' "$1" >&2
+      ;;
+    *)
+      break
   esac
+  shift
 done
 
-if [[ -z "$FORCE" && "$(docker-compose ps | wc -l)" -gt 2 ]] ; then
-  echo "You should probably stop services before running this command"
-  echo "(use -f to skip this check)"
+if ! docker info &> /dev/null; then
+  echo "Docker is not running! Start docker daemon and try again."
   exit 1
 fi
 
+if [[ "$(docker-compose top | wc -l)" -gt 0 ]] ; then
+  echo "You should probably stop docker containers before running this command"
+  prompt "Would you like to attempt to stop containers with docker-compose stop? [y/n]" stop
+  if [[ ${stop:-n} == 'y' ]]; then
+    docker-compose stop
+  else
+    echo "Continuing with docker containers running, this may cause errors."
+  fi
+fi
+
 if [ -f "docker-compose.override.yml" ]; then
   echo "docker-compose.override.yml exists, skipping copy of default configuration"
 else
   echo "Copying default configuration from config/docker-compose.override.yml.example to docker-compose.override.yml"
   cp config/docker-compose.override.yml.example docker-compose.override.yml
 fi
+echo ""
 
-[[ -z "$SKIP_CODE" ]] && ./script/canvas_update -n data
-[[ -z "$SKIP_BUILD" ]] && docker-compose build --pull
-if [[ -z "$SKIP_BUILD" ]] ; then
-  # assets are currently compiled during dc build --pull
-  docker-compose run --rm web ./script/canvas_update -n code -n assets
-else
-  docker-compose run --rm web ./script/canvas_update -n code
-fi
+create_log_file
+intro_message "Docker Dev Update"
+[[ -n "$UPDATE_CODE" ]] && ./script/rebase_canvas_and_plugins.sh "${params[@]}"
+bundle_install_with_check
+install_node_packages
+compile_assets
+rake_db_migrate_dev_and_test

script/rebase_canvas_and_plugins.sh

@@ -34,7 +34,7 @@ while :; do
       SKIP_CANVAS=true
       ;;
     --skip-plugins)
-      if [ "$2" ]; then
+      if [[ "$2" ]] && [[ "$2" != --* ]]; then
         repos=$2
         IFS=',' read -r -a skip_repos <<< "$repos"
         shift
@@ -42,7 +42,7 @@ while :; do
         SKIP_PLUGINS=true
       fi
       ;;
-    -?*)
+    ?*)
       die 'ERROR: Unknown option: ' "$1" >&2
       ;;
     *)
@@ -122,8 +122,12 @@ function check_for_changes {
   if [ ${#stash_code[@]} -gt 0 ]; then
     printf -v joined '%s, ' "${stash_code[@]}"
     message "You have uncommitted changes in ${joined%, }." | tee -a "$LOG"
-    prompt "  Ok to run \"git stash push -m 'Stashed as part of rebase_canvas_and_plugins'\" for each repo above? [y/n]" run_stash
-    [[ ${run_stash:-n} != 'y' ]] && printf "\nStash or commit your changes then run this script again.\n" | tee -a "$LOG" && exit
+    prompt "  Ok to run \"git stash push -m 'Stashed as part of rebase_canvas_and_plugins'\" for each repo above? [y/n/skip]" run_stash
+    if [[ ${run_stash:-n} == 'skip' ]]; then
+      printf "\nSkipping stash, attempting to rebase with uncommitted changes.\n" | tee -a "$LOG" && return
+    elif [[ ${run_stash:-n} != 'y' ]]; then
+      printf "\nStash or commit your changes then run this script again.\n" | tee -a "$LOG" && exit
+    fi
     stash_repos "${stash_code[@]}"
   else
     echo_console_and_log "No uncommitted changes found. \o/"
@@ -140,9 +144,10 @@ function print_results {
     for repo in ${FAILED_REPOS[*]}; do
       echo_console_and_log "  $repo failed to rebase cleanly!"
     done
+    exit 1
   elif [ "${exit_code}" == 0 ]; then
-    # if skipped stashing, don't print anymore.
-    [[ ${run_stash:-n} != 'y' ]] && return
+    # if no stashing, don't print anymore.
+    [[ ${run_stash:-} == 'n' ]] && exit 1
     echo ""
     echo_console_and_log "\o/ SUCCESS!"
   elif [ "${exit_code}" != 130 ]; then
@@ -156,5 +161,5 @@ ensure_in_canvas_root_directory
 create_log_file
 intro_message "Rebase Canvas and Plugins"
 check_for_changes
-[[ -z "$SKIP_CANVAS" ]] && rebase_canvas
-[[ -z "$SKIP_PLUGINS" ]] && rebase_plugins
+[[ -n "$SKIP_CANVAS" ]] || rebase_canvas
+[[ -n "$SKIP_PLUGINS" ]] || rebase_plugins