Fetch things sooner on the dashboard

closes: CORE-2904 CORE-2905 CORE-2906

This will send the XHR requests for the dashcards and for all the stuff
in the planner dashboard sooner. As in, as soon as the html document 
comes back from the server, instead of waiting until all the js bundles
have downloaded and executed. This should remove a round-trip from
the dashboard page and you should notice the planner and dashcard
dashboards complete loading a lot faster. By sending the requests
sooner, our app servers can work on them while the browser is 
downloading all the rest of the JS that it needs to run the page. And 
by having our app servers work on that at the same time the browser 
works on the JS, the entire page will complete loading faster.

Test plan:
* go to the planner dashboard
* you should see 4 requests with the type “fetch” in the network panel
  in the waterfall right at the same time the html document is returned
* you should not see any more XHRs sent from axios on page load for the
  planner dashboard (it should use the prefetched ones)
* in a prod environment, where you have multiple app server processes
  handling requests, the planner dashboard should complete loading a lot
  faster

* In that same prod like environment, go to the dashcard dashboard
* same thing, it should issue a “fetch” request right with the html
  document comes back from the server and not issue an second
  XHR once the JS is loaded.
* it should be complete loading significantly faster too.


Change-Id: I0503c1a4d913fd1baa4dad22b9a88333ff747c0d
Reviewed-on: https://gerrit.instructure.com/192161
Tested-by: Jenkins
QA-Review: Mysti Lilla <mysti@instructure.com>
Product-Review: Mysti Lilla <mysti@instructure.com>
Reviewed-by: Brent Burgoyne <bburgoyne@instructure.com>

.gitignore

@@ -61,8 +61,8 @@ docker-compose.local.*
 
 # sub-packages
 /packages/*/node_modules
-/packages/canvas-planner/lib
-/packages/canvas-planner/es
+/packages/*/lib
+/packages/*/es
 /packages/canvas-planner/coverage/
 /packages/canvas-planner/.babel-cache
 

app/controllers/users_controller.rb

@@ -623,7 +623,7 @@ class UsersController < ApplicationController
       end
     end
 
-    render :layout => false
+    render :formats => 'html', :layout => false
   end
 
   def toggle_hide_dashcard_color_overlays

app/helpers/application_helper.rb

@@ -1209,4 +1209,14 @@ module ApplicationHelper
     # stub
   end
 
+  def prefetch_xhr(url, id: nil, options: {})
+    id ||= url
+    opts = {
+      credentials: 'same-origin',
+      headers: {
+        Accept: 'application/json+canvas-string-ids, application/json'
+      }
+    }.deep_merge(options)
+    javascript_tag "(window.prefetched_xhrs = (window.prefetched_xhrs || {}))[#{id.to_json}] = fetch(#{url.to_json}, #{opts.to_json})"
+  end
 end

app/jsx/bundles/dashboard_card.js

@@ -20,6 +20,7 @@ import React from 'react'
 import ReactDOM from 'react-dom'
 import getDroppableDashboardCardBox from '../dashboard_card/getDroppableDashboardCardBox'
 import axios from 'axios'
+import {asJson, getPrefetchedXHR} from '@instructure/js-utils'
 
 let promiseToGetDashboardCards
 
@@ -45,7 +46,9 @@ export default function loadCardDashboard() {
   if (cachedCards) render(JSON.parse(cachedCards))
 
   if (!promiseToGetDashboardCards) {
-    promiseToGetDashboardCards = axios.get('/api/v1/dashboard/dashboard_cards').then(({data}) => data)
+    const url = '/api/v1/dashboard/dashboard_cards'
+    promiseToGetDashboardCards = asJson(getPrefetchedXHR(url)) || axios.get(url).then(({data}) => data)
+
     promiseToGetDashboardCards.then(dashboardCards =>
       sessionStorage.setItem(sessionStorageKey, JSON.stringify(dashboardCards))
     )

app/jsx/dashboard/DashboardHeader.js

@@ -28,6 +28,7 @@ import apiUserContent from 'compiled/str/apiUserContent'
 import DashboardOptionsMenu from '../dashboard_card/DashboardOptionsMenu';
 import loadCardDashboard from '../bundles/dashboard_card'
 import $ from 'jquery'
+import {asText, getPrefetchedXHR} from '@instructure/js-utils'
 import 'jquery.disableWhileLoading'
 
 const [show, hide] = ['block', 'none'].map(displayVal => id => {
@@ -221,7 +222,8 @@ function showTodoList () {
   if (ENV.DASHBOARD_SIDEBAR_URL) {
     const rightSide = $('#right-side')
     const promiseToGetNewCourseForm = import('compiled/util/newCourseForm')
-    const promiseToGetHtml = $.get(ENV.DASHBOARD_SIDEBAR_URL)
+    const promiseToGetHtml = asText(getPrefetchedXHR(ENV.DASHBOARD_SIDEBAR_URL)) || $.get(ENV.DASHBOARD_SIDEBAR_URL)
+
     rightSide.disableWhileLoading(
       Promise.all([promiseToGetNewCourseForm, promiseToGetHtml]).then(([{default: newCourseForm}, html]) => {
         // inject the erb html we got from the server

app/jsx/discussions/apiClient.js

@@ -17,32 +17,14 @@
  */
 
 import axios from 'axios'
-
-// In the the index.html.erb view for this page, we fire of `fetch` requests for all the
-// get requests for all the discusisons we're going to render. We do this so they
-// can start loading then and not have to wait until this JS file is loaded to start
-// fetching. But since they are just raw `fetch` responses, we need to massage them
-// into something that looks like an axios response, since that is what everything
-// here is designed to deal with
-let axiosResponses
-function getFetchRequests() {
-  return (
-    axiosResponses ||
-    (axiosResponses = (window.preloadedDiscussionTopicFetchRequests || []).map(fetchRequest => {
-      return fetchRequest.then(res => {
-        return res.json().then(json => {
-          return {
-            data: json,
-            headers: {link: res.headers.get('Link')}
-          }
-        })
-      })
-    }))
-  )
-}
+import {asAxios, getPrefetchedXHR} from '@instructure/js-utils'
 
 export function getDiscussions({contextType, contextId}, {page}) {
-  return getFetchRequests()[page - 1]
+  // In the the index.html.erb view for this page, we use prefetch_xhr to fire off
+  // `fetch` requests for all the discusisons we're going to render. We do this
+  // so they can start loading then and not have to wait until this JS file is
+  // loaded to start fetching.
+  return asAxios(getPrefetchedXHR(`prefetched_discussion_topic_page_${page - 1}`))
 }
 
 export function updateDiscussion({contextType, contextId}, discussion, updatedFields) {

app/views/discussion_topics/index.html.erb

@@ -24,19 +24,10 @@
 
   js_bundle :discussion_topics_index_v2
   css_bundle :discussions_index
-
 %>
 <% content_for :stylesheets do # we put this in :stylesheets so it is rendered early, in the <head> %>
-  <script>
-    preloadedDiscussionTopicFetchRequests = <%= raw(@discussion_topics_urls_to_prefetch.to_json) %>
-      .map(function(url) {
-        return fetch(url, {
-          credentials: 'same-origin',
-          headers: {
-            'Accept': 'application/json+canvas-string-ids, application/json'
-          }
-        })
-      });
-  </script>
+  <% @discussion_topics_urls_to_prefetch.each_with_index do |url, i| %>
+    <%= prefetch_xhr(url, id: "prefetched_discussion_topic_page_#{i}") %>
+  <% end %>
 <% end %>
 

app/views/shared/_dashboard_card.html.erb

@@ -22,9 +22,15 @@ default_number_of_fake_dashcards_to_show = 5
 number_of_fake_cards_to_show =
   Rails.cache.read(['last_known_dashboard_cards_count', @current_user.global_id].cache_key) ||
   default_number_of_fake_dashcards_to_show
-%>
 
-<div id="DashboardCard_Container" style="display: <%= user_dashboard_view == 'cards' ? 'block' : 'none' %>">
+render_on_pageload = user_dashboard_view == 'cards'
+%>
+<% if render_on_pageload %>
+  <%# fire off these `fetch` requests now so they are ready sooner %>
+  <%= prefetch_xhr('/api/v1/dashboard/dashboard_cards') %>
+  <%= prefetch_xhr(dashboard_sidebar_url) %>
+<% end %>
+<div id="DashboardCard_Container" style="display: <%= render_on_pageload ? 'block' : 'none' %>">
   <div class="ic-DashboardCard__box">
     <% for i in 1..number_of_fake_cards_to_show do %>
       <div class="ic-DashboardCard">

app/views/users/user_dashboard.html.erb

@@ -17,9 +17,22 @@
 %>
 
 <% content_for :page_title do %><%= t('Dashboard') %><% end %>
-<% css_bundle :dashboard %>
-<% js_bundle :dashboard %>
-<% @body_classes << "dashboard-is-planner" if show_planner? %>
+<%
+  css_bundle :dashboard
+  js_bundle :dashboard
+
+  if show_planner?
+    @body_classes << "dashboard-is-planner"
+
+    # fire off these `fetch` requests now so they are ready sooner
+    six_months_ago = Time.now.utc.at_beginning_of_day.months_ago(6).iso8601(3)
+    beginning_of_day = Time.zone.now.at_beginning_of_day.utc.iso8601(3)
+%>
+  <%= prefetch_xhr("/api/v1/planner/items?start_date=#{six_months_ago}&filter=new_activity&order=asc") %>
+  <%= prefetch_xhr("/api/v1/planner/items?end_date=#{beginning_of_day}&order=desc&per_page=1") %>
+  <%= prefetch_xhr("/api/v1/planner/items?start_date=#{beginning_of_day}") %>
+  <%= prefetch_xhr('/api/v1/users/self/missing_submissions?include[]=planner_overrides&filter[]=submittable') %>
+<% end %>
 
 <% content_for :auto_discovery do %>
   <% if @current_user %>

packages/canvas-planner/src/actions/__tests__/loading-actions.spec.js

@@ -70,8 +70,7 @@ describe('api actions', () => {
         fromMoment, getState: () => ({loading: {}}),
       });
       return moxiosWait(request => {
-        expect(request.config.url).toBe('/api/v1/planner/items');
-        expect(request.config.params.start_date).toBe(fromMoment.toISOString());
+        expect(request.config.url).toBe(`/api/v1/planner/items?start_date=${fromMoment.toISOString()}`);
       });
     });
 
@@ -91,9 +90,7 @@ describe('api actions', () => {
         fromMoment, intoThePast: true, getState: () => ({loading: {}}),
       });
       return moxiosWait(request => {
-        expect(request.config.url).toBe('/api/v1/planner/items');
-        expect(request.config.params.end_date).toBe(fromMoment.toISOString());
-        expect(request.config.params.order).toBe('desc');
+        expect(request.config.url).toBe(`/api/v1/planner/items?end_date=${fromMoment.toISOString()}&order=desc`);
       });
     });
 
@@ -144,9 +141,7 @@ describe('api actions', () => {
       const mockMoment = moment.tz('Asia/Tokyo').startOf('day');
       Actions.getFirstNewActivityDate(mockMoment)(mockDispatch, getBasicState);
       return moxiosWait(request => {
-        expect(request.config.params.filter).toBe('new_activity');
-        expect(request.config.params.start_date).toBe(mockMoment.subtract(6, 'months').toISOString());
-        expect(request.config.params.order).toBe('asc');
+        expect(request.url).toBe(`/api/v1/planner/items?start_date=${mockMoment.subtract(6, 'months').toISOString()}&filter=new_activity&order=asc`);
       });
     });
 

packages/canvas-planner/src/actions/index.js

@@ -17,6 +17,7 @@
  */
 import { createActions, createAction } from 'redux-actions';
 import axios from 'axios';
+import {asAxios, getPrefetchedXHR} from '@instructure/js-utils';
 import configureAxios from '../utilities/configureAxios';
 import { alert } from '../utilities/alertUtils';
 import formatMessage from '../format-message';
@@ -110,15 +111,12 @@ export const getInitialOpportunities = () => {
   return (dispatch, getState) => {
     dispatch(startLoadingOpportunities());
 
-    axios({
-      method: 'get',
-      url: getState().opportunities.nextUrl || '/api/v1/users/self/missing_submissions?include[]=planner_overrides&filter[]=submittable',
-    }).then(response => {
-      if(parseLinkHeader(response.headers.link).next) {
-        dispatch(addOpportunities({items: response.data, nextUrl: parseLinkHeader(response.headers.link).next.url }));
-      }else {
-        dispatch(addOpportunities({items: response.data, nextUrl: null}));
-      }
+    const url = getState().opportunities.nextUrl || '/api/v1/users/self/missing_submissions?include[]=planner_overrides&filter[]=submittable'
+    const request = asAxios(getPrefetchedXHR(url)) || axios({method: 'get', url})
+
+    request.then(response => {
+      const next = parseLinkHeader(response.headers.link).next;
+      dispatch(addOpportunities({items: response.data, nextUrl: (next ? next.url : null)}));
     }).catch(() => alert(formatMessage('Failed to load opportunities'), true));
   };
 };

packages/canvas-planner/src/actions/loading-actions.js

@@ -18,6 +18,8 @@
 
 import { createActions, createAction } from 'redux-actions';
 import axios from 'axios';
+import buildURL from 'axios/lib/helpers/buildURL.js';
+import {asAxios, getPrefetchedXHR} from '@instructure/js-utils';
 import { transformApiToInternalItem } from '../utilities/apiUtils';
 import { alert } from '../utilities/alertUtils';
 import formatMessage from '../format-message';
@@ -84,11 +86,11 @@ export function getFirstNewActivityDate (fromMoment) {
   // specifically so we know what the very oldest new activity is
   return (dispatch, getState) => {
     fromMoment = fromMoment.clone().subtract(6, 'months');
-    return axios.get('/api/v1/planner/items', { params: {
-      start_date: fromMoment.toISOString(),
-      filter: 'new_activity',
-      order: 'asc'
-    }}).then(response => {
+
+    const url = `/api/v1/planner/items?start_date=${fromMoment.toISOString()}&filter=new_activity&order=asc`
+    const request = asAxios(getPrefetchedXHR(url)) || axios.get(url)
+
+    return request.then(response => {
       if (response.data.length) {
         const first = transformApiToInternalItem(response.data[0], getState().courses, getState().groups, getState().timeZone);
         dispatch(foundFirstNewActivityDate(first.dateBucketMoment));
@@ -156,10 +158,11 @@ export const loadPastUntilToday = () => (dispatch, getState) => {
 
 
 export function sendFetchRequest (loadingOptions) {
-  return axios.get(...fetchParams(loadingOptions))
-    .then(response => handleFetchResponse(loadingOptions, response))
-    // no .catch: it's up to the sagas to handle errors
-  ;
+  const [urlPrefix, {params}] = fetchParams(loadingOptions)
+  const url = buildURL(urlPrefix, params)
+  const request = asAxios(getPrefetchedXHR(url)) ||  axios.get(url)
+  return request.then(response => handleFetchResponse(loadingOptions, response))
+  // no .catch: it's up to the sagas to handle errors
 }
 
 function fetchParams (loadingOptions) {

packages/js-utils/package.json

@@ -0,0 +1,22 @@
+{
+  "name": "@instructure/js-utils",
+  "version": "1.0.0",
+  "description": "A collection of utilities/helpers used by projects in the canvas/edu/instructure ecosystem.",
+  "author": "Instructure, Inc.",
+  "main": "lib/index.js",
+  "module": "es/index.js",
+  "sideEffects": false,
+  "scripts": {
+    "test": "echo 'no tests yet...'",
+    "build:canvas": "babel --root-mode upward src -d es & JEST_WORKER_ID=true babel --root-mode upward src -d lib"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/instructure/canvas-lms.git"
+  },
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/instructure/canvas-lms/issues"
+  },
+  "homepage": "https://github.com/instructure/canvas-lms#readme"
+}

packages/js-utils/src/index.js

@@ -0,0 +1 @@
+export * from './prefetched_xhrs'

packages/js-utils/src/prefetched_xhrs.js

@@ -0,0 +1,43 @@
+
+
+// These are helpful methods you can use along side the ruby ApplicationHelper::prefetch_xhr helper method in canvas
+
+export function getPrefetchedXHR(id){
+  return window.prefetched_xhrs && window.prefetched_xhrs[id]
+}
+
+/**
+ * Transforms a `fetch` request into something that looks like an `axios` response
+ * with a `.data` and `.headers` property, so you can pass it to our parseLinkHeaders stuff
+ *
+ * @param {Promise<Response>} fetchRequest
+ * @returns {Promise<import("axios").AxiosResponse>}
+ */
+export function asAxios(fetchRequest) {
+  if (!fetchRequest) return
+  return fetchRequest.then(res =>
+    res.json().then(data => ({data, headers: {link: res.headers.get('Link')}}))
+  )
+}
+
+/**
+ * Takes a `fetch` request and returns a promise of the json data of the response
+ *
+ * @param {Promise<Response>} fetchRequest
+ * @returns {Promise<JSON_data>}
+ */
+export function asJson(fetchRequest) {
+  if (!fetchRequest) return
+  return fetchRequest.then(res => res.json())
+}
+
+/**
+ * Takes a `fetch` request and returns a promise of the text of the response
+ *
+ * @param {Promise<Response>} fetchRequest
+ * @returns {Promise<USVString>}
+ */
+export function asText(fetchRequest) {
+  if (!fetchRequest) return
+  return fetchRequest.then(res => res.text())
+}

spec/helpers/application_helper_spec.rb

@@ -1032,6 +1032,18 @@ describe ApplicationHelper do
     end
   end
 
+  describe "#prefetch_xhr" do
+    it "inserts a script tag that will have a `fetch` call with the right id, url, and options" do
+      expect(prefetch_xhr('some_url', id: 'some_id', options: {headers: {"x-some-header": "some-value"}})).to eq(
+"<script>
+//<![CDATA[
+(window.prefetched_xhrs = (window.prefetched_xhrs || {}))[\"some_id\"] = fetch(\"some_url\", {\"credentials\":\"same-origin\",\"headers\":{\"Accept\":\"application/json+canvas-string-ids, application/json\",\"x-some-header\":\"some-value\"}})
+//]]>
+</script>"
+      )
+    end
+  end
+
   describe "#alt_text_for_login_logo" do
     before :each do
       @domain_root_account = Account.default

spec/views/discussion_topics/index.html.erb_spec.rb

@@ -24,6 +24,7 @@ describe "/discussion_topics/index" do
     course_with_teacher
     view_context(@course, @user)
     assign(:body_classes, [])
+    assign(:discussion_topics_urls_to_prefetch, [])
     render "discussion_topics/index"
     expect(response).not_to be_nil
   end