diff --git a/tools/offline/README.md b/tools/offline/README.md new file mode 100644 index 0000000000..b8a6f3bcd2 --- /dev/null +++ b/tools/offline/README.md @@ -0,0 +1,70 @@ +# Building LAMMPS and its documentation on offline systems + +In some situations it might be necessary to build LAMMPS on a system without +internet. The scripts in this folder allow you to preload external dependencies +for both the documentation build for building with CMake into a folder and then +use that folder on an offline system. + +It does so by + +1.) Downloading necessary pip packages +2.) Cloning git repositories +3.) Downloading tarballs + +As of April 2021, all of these downloads make up around 600MB. By +default, it will download everything into $HOME/.cache/lammps, but this can be +changed with the ``LAMMPS_CACHING_DIR`` environment variable. + +Once the caches have been initialized, they can be used for building +LAMMPS documentation or compiling using CMake on an offline system. + +The ``use_caches.sh`` must be sourced into the current shell to initialize the +offline build environment. Note that it must use the same ``LAMMPS_CACHING_DIR``. +This script does the following: + +1.) Sets up environment variables that modify the behavior of both pip and git +2.) Starts a simple local HTTP server to host files for CMake + +Afterwards, it will print out instruction on how to modify the CMake command +line to make sure it uses the local HTTP server. + +To undo the environment changes and shutdown the HTTP server, run the +``deactivate_caches`` command. + +## Examples + +For all of the examples below, you first need to create the cache (which requires internet). + +```bash +./tools/offline/init_caches.sh +``` + +Afterwards, you can disconnect or copy the contents of the +``LAMMPS_CACHING_DIR`` folder to an offline system. + +### Documentation + +```bash +# if LAMMPS_CACHING_DIR is different from default, make sure to set it first +# export LAMMPS_CACHING_DIR=path/to/folder +source tools/offline/use_caches.sh +cd doc/ +make html + +deactivate_caches +``` + +### CMake Build + +```bash +# if LAMMPS_CACHING_DIR is different from default, make sure to set it first +# export LAMMPS_CACHING_DIR=path/to/folder +source tools/offline/use_caches.sh + +mkdir build +cd build +cmake -D LAMMPS_DOWNLOADS_URL=${HTTP_CACHE_URL} -C ${LAMMPS_HTTP_CACHE_CONFIG} -C ../cmake/presets/most.cmake ../cmake +make -j 8 + +deactivate_caches +``` diff --git a/tools/offline/init_caches.sh b/tools/offline/init_caches.sh new file mode 100755 index 0000000000..692ebc257a --- /dev/null +++ b/tools/offline/init_caches.sh @@ -0,0 +1,45 @@ +#!/bin/bash +echo "##############################################################################" +echo "Initializing LAMMPS offline compilation environment" +echo "##############################################################################" + +if [ -z "${LAMMPS_CACHING_DIR}" ] +then + export LAMMPS_CACHING_DIR=$HOME/.cache/lammps + echo "environment variable LAMMPS_CACHING_DIR not set" + echo "Using default $LAMMPS_CACHING_DIR as cache directory..." +else + echo "Using $LAMMPS_CACHING_DIR as cache directory..." +fi + +SCRIPT_DIR="$(dirname "$(realpath "$0")")" +CACHE_SCRIPTS_DIR=${SCRIPT_DIR}/scripts + +if [ -z "${LAMMPS_DIR}" ] +then + export LAMMPS_DIR=$(realpath $SCRIPT_DIR/../../) + echo "environment variable LAMMPS_DIR not set" + echo "Using default $LAMMPS_DIR as LAMMPS distribution base directory..." +else + echo "Using $LAMMPS_DIR as LAMMPS distribution base directory..." +fi + +export GITHUB_PROXY_DIR=$LAMMPS_CACHING_DIR/github +export LOGGING_DIR=$LAMMPS_CACHING_DIR/logs +export PIP_CACHE_DIR=$LAMMPS_CACHING_DIR/pip +export HTTP_CACHE_DIR=$LAMMPS_CACHING_DIR/http + +mkdir -p $GITHUB_PROXY_DIR +mkdir -p $LOGGING_DIR +mkdir -p $PIP_CACHE_DIR +mkdir -p $HTTP_CACHE_DIR + +${CACHE_SCRIPTS_DIR}/init_pip_cache.sh +${CACHE_SCRIPTS_DIR}/init_git_cache.sh +${CACHE_SCRIPTS_DIR}/init_http_cache.sh +echo "##############################################################################" +echo +echo "To activate:" +echo "source ${SCRIPT_DIR}/use_caches.sh" +echo +echo "##############################################################################" diff --git a/tools/offline/scripts/init_git_cache.sh b/tools/offline/scripts/init_git_cache.sh new file mode 100755 index 0000000000..d33c4276df --- /dev/null +++ b/tools/offline/scripts/init_git_cache.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +if [ -z "${GITHUB_PROXY_DIR}" ] +then + echo "Must set GITHUB_PROXY_DIR environment variable" + exit 1 +fi + +mkdir -p $GITHUB_PROXY_DIR +cd $GITHUB_PROXY_DIR + +PROJECTS=( + akohlmey/sphinx-fortran + mathjax/MathJax +) + +for project in ${PROJECTS[@]} +do + GH_NAMESPACE=$(dirname $project) + GH_PROJECT=$(basename $project) + mkdir -p $GH_NAMESPACE + git clone --bare https://github.com/$GH_NAMESPACE/$GH_PROJECT.git $GH_NAMESPACE/$GH_PROJECT.git +done diff --git a/tools/offline/scripts/init_http_cache.sh b/tools/offline/scripts/init_http_cache.sh new file mode 100755 index 0000000000..c5e4d3dde7 --- /dev/null +++ b/tools/offline/scripts/init_http_cache.sh @@ -0,0 +1,103 @@ +#!/bin/bash + +if [ -z "${HTTP_CACHE_DIR}" ] +then + echo "Must set HTTP_CACHE_DIR environment variable" + exit 1 +fi + +mkdir -p $HTTP_CACHE_DIR +mkdir -p $HTTP_CACHE_DIR/potentials +mkdir -p $HTTP_CACHE_DIR/thirdparty +cd $HTTP_CACHE_DIR + +LAMMPS_DOWNLOADS_URL="https://download.lammps.org" +LAMMPS_POTENTIALS_URL="${LAMMPS_DOWNLOADS_URL}/potentials" +LAMMPS_THIRDPARTY_URL="${LAMMPS_DOWNLOADS_URL}/thirdparty" + +############################################################################### +# potentials +POTENTIALS=( + C_10_10.mesocnt.028de73ec828b7830d762702eda571c1 + TABTP_10_10.mesont.744a739da49ad5e78492c1fc9fd9f8c1 + C_10_10.mesocnt + TABTP_10_10.mesont +) + +echo "Dowloading potentials..." +for p in ${POTENTIALS[@]} +do + if [ ! -f $HTTP_CACHE_DIR/potentials/$p ] + then + wget -O $HTTP_CACHE_DIR/potentials/$p $LAMMPS_POTENTIALS_URL/$p + fi +done + +############################################################################### +# thirdparty code +echo "Dowloading thirdparty tarballs..." + +MPICH2_WIN64_DEVEL_URL="${LAMMPS_THIRDPARTY_URL}/mpich2-win64-devel.tar.gz" +MPICH2_WIN32_DEVEL_URL="${LAMMPS_THIRDPARTY_URL}/mpich2-win32-devel.tar.gz" +VORO_URL="${LAMMPS_THIRDPARTY_URL}/voro++-0.4.6.tar.gz" +OPENCL_LOADER_URL="${LAMMPS_THIRDPARTY_URL}/opencl-loader-2020.12.18.tar.gz" +SCAFACOS_FIX_URL="${LAMMPS_THIRDPARTY_URL}/scafacos-1.0.1-fix.diff" +GTEST_URL="https://github.com/google/googletest/archive/release-1.10.0.tar.gz" +YAML_URL="https://pyyaml.org/download/libyaml/yaml-0.2.5.tar.gz" +MATHJAX_URL="https://github.com/mathjax/MathJax/archive/3.1.2.tar.gz" +EIGEN3_URL="https://gitlab.com/libeigen/eigen/-/archive/3.3.7/eigen-3.3.7.tar.gz" +CUB_URL="https://github.com/NVlabs/cub/archive/1.12.0.tar.gz" +KOKKOS_URL="https://github.com/kokkos/kokkos/archive/3.3.01.tar.gz" +KIM_URL="https://s3.openkim.org/kim-api/kim-api-2.2.1.txz" +MSCG_URL="https://github.com/uchicago-voth/MSCG-release/archive/1.7.3.1.tar.gz" +PLUMED_URL="https://github.com/plumed/plumed2/releases/download/v2.7.0/plumed-src-2.7.0.tgz" +LATTE_URL="https://github.com/lanl/LATTE/archive/v1.2.2.tar.gz" +SCAFACOS_URL="https://github.com/scafacos/scafacos/releases/download/v1.0.1/scafacos-1.0.1.tar.gz" + +GTEST_FILENAME="gtest-1.10.0.tar.gz" +MATHJAX_FILENAME="mathjax-3.1.2.tar.gz" +CUB_FILENAME="cub-1.12.0.tar.gz" +KOKKOS_FILENAME="kokkos-3.3.01.tar.gz" +MSCG_FILENAME="mscg-1.7.3.1.tar.gz" +LATTE_FILENAME="latte-1.2.2.tar.gz" + +TARBALLS=( + MPICH2_WIN64_DEVEL_URL + MPICH2_WIN32_DEVEL_URL + VORO_URL + OPENCL_LOADER_URL + SCAFACOS_FIX_URL + GTEST_URL + YAML_URL + MATHJAX_URL + EIGEN3_URL + CUB_URL + KOKKOS_URL + KIM_URL + MSCG_URL + PLUMED_URL + LATTE_URL + SCAFACOS_URL +) + +############################################################################### +# generate proxy cmake file to trick CMake to download from local HTTP server +echo "# auto-generated proxy preset file" > $HTTP_CACHE_DIR/proxy.cmake + +for t in ${TARBALLS[@]} +do + FILENAME_VAR=${t/_URL/_FILENAME} + if [ -z ${!FILENAME_VAR} ] + then + filename=$(basename ${!t}) + else + filename=${!FILENAME_VAR} + fi + + if [ ! -f $HTTP_CACHE_DIR/thirdparty/$filename ] + then + wget -O $HTTP_CACHE_DIR/thirdparty/$filename ${!t} + fi + + echo "set(${t} \"\${LAMMPS_DOWNLOADS_URL}/thirdparty/$filename\" CACHE STRING \"\" FORCE)" >> $HTTP_CACHE_DIR/proxy.cmake +done diff --git a/tools/offline/scripts/init_pip_cache.sh b/tools/offline/scripts/init_pip_cache.sh new file mode 100755 index 0000000000..56160489cb --- /dev/null +++ b/tools/offline/scripts/init_pip_cache.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +if [ -z "${LOGGING_DIR}" ] +then + echo "Must set LOGGING_DIR environment variable" + exit 1 +fi + +if [ -z "${PIP_CACHE_DIR}" ] +then + echo "Must set PIP_CACHE_DIR environment variable" + exit 1 +fi + +set -x + +mkdir -p $PIP_CACHE_DIR + +# download packages that might be needed +cd $PIP_CACHE_DIR +pip3 download pip setuptools wheel +pip3 download -r $LAMMPS_DIR/doc/utils/requirements.txt diff --git a/tools/offline/scripts/use_git_cache.sh b/tools/offline/scripts/use_git_cache.sh new file mode 100644 index 0000000000..049637725a --- /dev/null +++ b/tools/offline/scripts/use_git_cache.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# needs to be sourced +if [ -z "${GITHUB_PROXY_DIR}" ] +then + echo "Must set GITHUB_PROXY_DIR environment variable" + exit 1 +fi + +export GIT_CONFIG_COUNT=1 +export GIT_CONFIG_KEY_0=url.$GITHUB_PROXY_DIR/.insteadOf +export GIT_CONFIG_VALUE_0=git://github.com/ + +echo "Redirecting git://github.com urls to local cache..." + +function deactivate_git_cache { + echo "Removing git://github.com redirect..." + unset GIT_CONFIG_COUNT + unset GIT_CONFIG_KEY_0 + unset GIT_CONFIG_VALUE_0 +} diff --git a/tools/offline/scripts/use_http_cache.sh b/tools/offline/scripts/use_http_cache.sh new file mode 100644 index 0000000000..717ab9f8b4 --- /dev/null +++ b/tools/offline/scripts/use_http_cache.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +if [ -z "${HTTP_CACHE_DIR}" ] +then + echo "Must set HTTP_CACHE_DIR environment variable" + exit 1 +fi + +if [ -z "${HTTP_CACHE_PORT}" ] +then + HTTP_CACHE_PORT=8080 +fi + +until ! lsof -Pi :$HTTP_CACHE_PORT -sTCP:LISTEN -t >/dev/null +do + echo "Port ${HTTP_CACHE_PORT} already in use, trying another..." + ((HTTP_CACHE_PORT=HTTP_CACHE_PORT+1)) + sleep 1 +done + +if [ -z "${LOGGING_DIR}" ] +then + echo "Must set LOGGING_DIR environment variable" + exit 1 +fi + +python3 -m http.server $HTTP_CACHE_PORT --directory $HTTP_CACHE_DIR 2>&1 > ${LOGGING_DIR}/http.log & +export HTTP_CACHE_PID=$! + +export HTTP_CACHE_URL=http://localhost:$HTTP_CACHE_PORT +export LAMMPS_HTTP_CACHE_CONFIG=$HTTP_CACHE_DIR/proxy.cmake +echo "Running local HTTP cache server on $HTTP_CACHE_URL (pid: $HTTP_CACHE_PID)" + +function deactivate_http_cache { + echo "Shutting down HTTP cache server..." + kill $HTTP_CACHE_PID + unset HTTP_CACHE_PID + unset LAMMPS_HTTP_CACHE_CONFIG +} diff --git a/tools/offline/scripts/use_pip_cache.sh b/tools/offline/scripts/use_pip_cache.sh new file mode 100644 index 0000000000..f122aacd33 --- /dev/null +++ b/tools/offline/scripts/use_pip_cache.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# needs to be sourced + +if [ -z "${PIP_CACHE_DIR}" ] +then + echo "Must set PIP_CACHE_DIR environment variable" + exit 1 +fi + +export PIP_NO_INDEX=1 +export PIP_FIND_LINKS=$PIP_CACHE_DIR + +echo "Disabling pip index and use local cache directory..." + +function deactivate_pip_cache { + echo "Removing local pip cache configuration..." + unset PIP_NO_INDEX + unset PIP_FIND_LINKS +} diff --git a/tools/offline/use_caches.sh b/tools/offline/use_caches.sh new file mode 100644 index 0000000000..64e7d7a875 --- /dev/null +++ b/tools/offline/use_caches.sh @@ -0,0 +1,79 @@ +#!/bin/bash + +if [ -z "${LAMMPS_CACHING_DIR}" ] +then + export LAMMPS_CACHING_DIR=$HOME/.cache/lammps + echo "environment variable LAMMPS_CACHING_DIR not set" + echo "Using default $LAMMPS_CACHING_DIR as cache directory..." +else + echo "Using $LAMMPS_CACHING_DIR as cache directory..." +fi + +if test -n "$BASH" ; then script=$BASH_SOURCE +else script=$0 +fi + +SCRIPT_DIR="$(dirname "$(realpath "$script")")" +CACHE_SCRIPTS_DIR=${SCRIPT_DIR}/scripts + +export GITHUB_PROXY_DIR=$LAMMPS_CACHING_DIR/github +export LOGGING_DIR=$LAMMPS_CACHING_DIR/logs +export PIP_CACHE_DIR=$LAMMPS_CACHING_DIR/pip +export HTTP_CACHE_DIR=$LAMMPS_CACHING_DIR/http + +if [ ! -d $GITHUB_PROXY_DIR ] +then + echo "GitHub proxy directory missing" + return +fi + +if [ ! -d $LOGGING_DIR ] +then + echo "Logging directory missing" + return +fi + +if [ ! -d $PIP_CACHE_DIR ] +then + echo "pip cache directory missing" + return +fi + +if [ ! -d $HTTP_CACHE_DIR ] +then + echo "HTTP cache directory missing" + return +fi + +echo "##############################################################################" +echo "Setting up LAMMPS offline compilation environment" +echo "##############################################################################" + +source ${CACHE_SCRIPTS_DIR}/use_git_cache.sh +source ${CACHE_SCRIPTS_DIR}/use_pip_cache.sh +source ${CACHE_SCRIPTS_DIR}/use_http_cache.sh + +echo "##############################################################################" +echo +echo "Prepend the following CMake options to your builds:" +echo +echo "-D LAMMPS_DOWNLOADS_URL=\${HTTP_CACHE_URL} -C \${LAMMPS_HTTP_CACHE_CONFIG}" +echo +echo "or" +echo +echo "-D LAMMPS_DOWNLOADS_URL=${HTTP_CACHE_URL} -C ${LAMMPS_HTTP_CACHE_CONFIG}" +echo +echo "pip installations and git clones (from git://) are automatically redirected" +echo +echo Use 'deactivate_caches' to revert changes +echo +echo "##############################################################################" + +function deactivate_caches { + deactivate_http_cache + deactivate_pip_cache + deactivate_git_cache + unset -f deactivate_http_cache + unset -f deactivate_pip_cache + unset -f deactivate_git_cache +}