forked from lijiext/lammps
Add scripts to simplify offline builds of docs and CMake
This commit is contained in:
Richard Berger
parent
00578e1b26
commit
8b410cf2bd
|
|
@ -0,0 +1,70 @@
|
|||
# Building LAMMPS and its documentation on offline systems
|
||||
|
||||
In some situations it might be necessary to build LAMMPS on a system without
|
||||
internet. The scripts in this folder allow you to preload external dependencies
|
||||
for both the documentation build for building with CMake into a folder and then
|
||||
use that folder on an offline system.
|
||||
|
||||
It does so by
|
||||
|
||||
1.) Downloading necessary pip packages
|
||||
2.) Cloning git repositories
|
||||
3.) Downloading tarballs
|
||||
|
||||
As of April 2021, all of these downloads make up around 600MB. By
|
||||
default, it will download everything into $HOME/.cache/lammps, but this can be
|
||||
changed with the ``LAMMPS_CACHING_DIR`` environment variable.
|
||||
|
||||
Once the caches have been initialized, they can be used for building
|
||||
LAMMPS documentation or compiling using CMake on an offline system.
|
||||
|
||||
The ``use_caches.sh`` must be sourced into the current shell to initialize the
|
||||
offline build environment. Note that it must use the same ``LAMMPS_CACHING_DIR``.
|
||||
This script does the following:
|
||||
|
||||
1.) Sets up environment variables that modify the behavior of both pip and git
|
||||
2.) Starts a simple local HTTP server to host files for CMake
|
||||
|
||||
Afterwards, it will print out instruction on how to modify the CMake command
|
||||
line to make sure it uses the local HTTP server.
|
||||
|
||||
To undo the environment changes and shutdown the HTTP server, run the
|
||||
``deactivate_caches`` command.
|
||||
|
||||
## Examples
|
||||
|
||||
For all of the examples below, you first need to create the cache (which requires internet).
|
||||
|
||||
```bash
|
||||
./tools/offline/init_caches.sh
|
||||
```
|
||||
|
||||
Afterwards, you can disconnect or copy the contents of the
|
||||
``LAMMPS_CACHING_DIR`` folder to an offline system.
|
||||
|
||||
### Documentation
|
||||
|
||||
```bash
|
||||
# if LAMMPS_CACHING_DIR is different from default, make sure to set it first
|
||||
# export LAMMPS_CACHING_DIR=path/to/folder
|
||||
source tools/offline/use_caches.sh
|
||||
cd doc/
|
||||
make html
|
||||
|
||||
deactivate_caches
|
||||
```
|
||||
|
||||
### CMake Build
|
||||
|
||||
```bash
|
||||
# if LAMMPS_CACHING_DIR is different from default, make sure to set it first
|
||||
# export LAMMPS_CACHING_DIR=path/to/folder
|
||||
source tools/offline/use_caches.sh
|
||||
|
||||
mkdir build
|
||||
cd build
|
||||
cmake -D LAMMPS_DOWNLOADS_URL=${HTTP_CACHE_URL} -C ${LAMMPS_HTTP_CACHE_CONFIG} -C ../cmake/presets/most.cmake ../cmake
|
||||
make -j 8
|
||||
|
||||
deactivate_caches
|
||||
```
|
||||
|
|
@ -0,0 +1,45 @@
|
|||
#!/bin/bash
|
||||
echo "##############################################################################"
|
||||
echo "Initializing LAMMPS offline compilation environment"
|
||||
echo "##############################################################################"
|
||||
|
||||
if [ -z "${LAMMPS_CACHING_DIR}" ]
|
||||
then
|
||||
export LAMMPS_CACHING_DIR=$HOME/.cache/lammps
|
||||
echo "environment variable LAMMPS_CACHING_DIR not set"
|
||||
echo "Using default $LAMMPS_CACHING_DIR as cache directory..."
|
||||
else
|
||||
echo "Using $LAMMPS_CACHING_DIR as cache directory..."
|
||||
fi
|
||||
|
||||
SCRIPT_DIR="$(dirname "$(realpath "$0")")"
|
||||
CACHE_SCRIPTS_DIR=${SCRIPT_DIR}/scripts
|
||||
|
||||
if [ -z "${LAMMPS_DIR}" ]
|
||||
then
|
||||
export LAMMPS_DIR=$(realpath $SCRIPT_DIR/../../)
|
||||
echo "environment variable LAMMPS_DIR not set"
|
||||
echo "Using default $LAMMPS_DIR as LAMMPS distribution base directory..."
|
||||
else
|
||||
echo "Using $LAMMPS_DIR as LAMMPS distribution base directory..."
|
||||
fi
|
||||
|
||||
export GITHUB_PROXY_DIR=$LAMMPS_CACHING_DIR/github
|
||||
export LOGGING_DIR=$LAMMPS_CACHING_DIR/logs
|
||||
export PIP_CACHE_DIR=$LAMMPS_CACHING_DIR/pip
|
||||
export HTTP_CACHE_DIR=$LAMMPS_CACHING_DIR/http
|
||||
|
||||
mkdir -p $GITHUB_PROXY_DIR
|
||||
mkdir -p $LOGGING_DIR
|
||||
mkdir -p $PIP_CACHE_DIR
|
||||
mkdir -p $HTTP_CACHE_DIR
|
||||
|
||||
${CACHE_SCRIPTS_DIR}/init_pip_cache.sh
|
||||
${CACHE_SCRIPTS_DIR}/init_git_cache.sh
|
||||
${CACHE_SCRIPTS_DIR}/init_http_cache.sh
|
||||
echo "##############################################################################"
|
||||
echo
|
||||
echo "To activate:"
|
||||
echo "source ${SCRIPT_DIR}/use_caches.sh"
|
||||
echo
|
||||
echo "##############################################################################"
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
#!/bin/bash
|
||||
|
||||
if [ -z "${GITHUB_PROXY_DIR}" ]
|
||||
then
|
||||
echo "Must set GITHUB_PROXY_DIR environment variable"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p $GITHUB_PROXY_DIR
|
||||
cd $GITHUB_PROXY_DIR
|
||||
|
||||
PROJECTS=(
|
||||
akohlmey/sphinx-fortran
|
||||
mathjax/MathJax
|
||||
)
|
||||
|
||||
for project in ${PROJECTS[@]}
|
||||
do
|
||||
GH_NAMESPACE=$(dirname $project)
|
||||
GH_PROJECT=$(basename $project)
|
||||
mkdir -p $GH_NAMESPACE
|
||||
git clone --bare https://github.com/$GH_NAMESPACE/$GH_PROJECT.git $GH_NAMESPACE/$GH_PROJECT.git
|
||||
done
|
||||
|
|
@ -0,0 +1,103 @@
|
|||
#!/bin/bash
|
||||
|
||||
if [ -z "${HTTP_CACHE_DIR}" ]
|
||||
then
|
||||
echo "Must set HTTP_CACHE_DIR environment variable"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p $HTTP_CACHE_DIR
|
||||
mkdir -p $HTTP_CACHE_DIR/potentials
|
||||
mkdir -p $HTTP_CACHE_DIR/thirdparty
|
||||
cd $HTTP_CACHE_DIR
|
||||
|
||||
LAMMPS_DOWNLOADS_URL="https://download.lammps.org"
|
||||
LAMMPS_POTENTIALS_URL="${LAMMPS_DOWNLOADS_URL}/potentials"
|
||||
LAMMPS_THIRDPARTY_URL="${LAMMPS_DOWNLOADS_URL}/thirdparty"
|
||||
|
||||
###############################################################################
|
||||
# potentials
|
||||
POTENTIALS=(
|
||||
C_10_10.mesocnt.028de73ec828b7830d762702eda571c1
|
||||
TABTP_10_10.mesont.744a739da49ad5e78492c1fc9fd9f8c1
|
||||
C_10_10.mesocnt
|
||||
TABTP_10_10.mesont
|
||||
)
|
||||
|
||||
echo "Dowloading potentials..."
|
||||
for p in ${POTENTIALS[@]}
|
||||
do
|
||||
if [ ! -f $HTTP_CACHE_DIR/potentials/$p ]
|
||||
then
|
||||
wget -O $HTTP_CACHE_DIR/potentials/$p $LAMMPS_POTENTIALS_URL/$p
|
||||
fi
|
||||
done
|
||||
|
||||
###############################################################################
|
||||
# thirdparty code
|
||||
echo "Dowloading thirdparty tarballs..."
|
||||
|
||||
MPICH2_WIN64_DEVEL_URL="${LAMMPS_THIRDPARTY_URL}/mpich2-win64-devel.tar.gz"
|
||||
MPICH2_WIN32_DEVEL_URL="${LAMMPS_THIRDPARTY_URL}/mpich2-win32-devel.tar.gz"
|
||||
VORO_URL="${LAMMPS_THIRDPARTY_URL}/voro++-0.4.6.tar.gz"
|
||||
OPENCL_LOADER_URL="${LAMMPS_THIRDPARTY_URL}/opencl-loader-2020.12.18.tar.gz"
|
||||
SCAFACOS_FIX_URL="${LAMMPS_THIRDPARTY_URL}/scafacos-1.0.1-fix.diff"
|
||||
GTEST_URL="https://github.com/google/googletest/archive/release-1.10.0.tar.gz"
|
||||
YAML_URL="https://pyyaml.org/download/libyaml/yaml-0.2.5.tar.gz"
|
||||
MATHJAX_URL="https://github.com/mathjax/MathJax/archive/3.1.2.tar.gz"
|
||||
EIGEN3_URL="https://gitlab.com/libeigen/eigen/-/archive/3.3.7/eigen-3.3.7.tar.gz"
|
||||
CUB_URL="https://github.com/NVlabs/cub/archive/1.12.0.tar.gz"
|
||||
KOKKOS_URL="https://github.com/kokkos/kokkos/archive/3.3.01.tar.gz"
|
||||
KIM_URL="https://s3.openkim.org/kim-api/kim-api-2.2.1.txz"
|
||||
MSCG_URL="https://github.com/uchicago-voth/MSCG-release/archive/1.7.3.1.tar.gz"
|
||||
PLUMED_URL="https://github.com/plumed/plumed2/releases/download/v2.7.0/plumed-src-2.7.0.tgz"
|
||||
LATTE_URL="https://github.com/lanl/LATTE/archive/v1.2.2.tar.gz"
|
||||
SCAFACOS_URL="https://github.com/scafacos/scafacos/releases/download/v1.0.1/scafacos-1.0.1.tar.gz"
|
||||
|
||||
GTEST_FILENAME="gtest-1.10.0.tar.gz"
|
||||
MATHJAX_FILENAME="mathjax-3.1.2.tar.gz"
|
||||
CUB_FILENAME="cub-1.12.0.tar.gz"
|
||||
KOKKOS_FILENAME="kokkos-3.3.01.tar.gz"
|
||||
MSCG_FILENAME="mscg-1.7.3.1.tar.gz"
|
||||
LATTE_FILENAME="latte-1.2.2.tar.gz"
|
||||
|
||||
TARBALLS=(
|
||||
MPICH2_WIN64_DEVEL_URL
|
||||
MPICH2_WIN32_DEVEL_URL
|
||||
VORO_URL
|
||||
OPENCL_LOADER_URL
|
||||
SCAFACOS_FIX_URL
|
||||
GTEST_URL
|
||||
YAML_URL
|
||||
MATHJAX_URL
|
||||
EIGEN3_URL
|
||||
CUB_URL
|
||||
KOKKOS_URL
|
||||
KIM_URL
|
||||
MSCG_URL
|
||||
PLUMED_URL
|
||||
LATTE_URL
|
||||
SCAFACOS_URL
|
||||
)
|
||||
|
||||
###############################################################################
|
||||
# generate proxy cmake file to trick CMake to download from local HTTP server
|
||||
echo "# auto-generated proxy preset file" > $HTTP_CACHE_DIR/proxy.cmake
|
||||
|
||||
for t in ${TARBALLS[@]}
|
||||
do
|
||||
FILENAME_VAR=${t/_URL/_FILENAME}
|
||||
if [ -z ${!FILENAME_VAR} ]
|
||||
then
|
||||
filename=$(basename ${!t})
|
||||
else
|
||||
filename=${!FILENAME_VAR}
|
||||
fi
|
||||
|
||||
if [ ! -f $HTTP_CACHE_DIR/thirdparty/$filename ]
|
||||
then
|
||||
wget -O $HTTP_CACHE_DIR/thirdparty/$filename ${!t}
|
||||
fi
|
||||
|
||||
echo "set(${t} \"\${LAMMPS_DOWNLOADS_URL}/thirdparty/$filename\" CACHE STRING \"\" FORCE)" >> $HTTP_CACHE_DIR/proxy.cmake
|
||||
done
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
#!/bin/bash
|
||||
|
||||
if [ -z "${LOGGING_DIR}" ]
|
||||
then
|
||||
echo "Must set LOGGING_DIR environment variable"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "${PIP_CACHE_DIR}" ]
|
||||
then
|
||||
echo "Must set PIP_CACHE_DIR environment variable"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
set -x
|
||||
|
||||
mkdir -p $PIP_CACHE_DIR
|
||||
|
||||
# download packages that might be needed
|
||||
cd $PIP_CACHE_DIR
|
||||
pip3 download pip setuptools wheel
|
||||
pip3 download -r $LAMMPS_DIR/doc/utils/requirements.txt
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
#!/bin/bash
|
||||
# needs to be sourced
|
||||
if [ -z "${GITHUB_PROXY_DIR}" ]
|
||||
then
|
||||
echo "Must set GITHUB_PROXY_DIR environment variable"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
export GIT_CONFIG_COUNT=1
|
||||
export GIT_CONFIG_KEY_0=url.$GITHUB_PROXY_DIR/.insteadOf
|
||||
export GIT_CONFIG_VALUE_0=git://github.com/
|
||||
|
||||
echo "Redirecting git://github.com urls to local cache..."
|
||||
|
||||
function deactivate_git_cache {
|
||||
echo "Removing git://github.com redirect..."
|
||||
unset GIT_CONFIG_COUNT
|
||||
unset GIT_CONFIG_KEY_0
|
||||
unset GIT_CONFIG_VALUE_0
|
||||
}
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
#!/bin/bash
|
||||
|
||||
if [ -z "${HTTP_CACHE_DIR}" ]
|
||||
then
|
||||
echo "Must set HTTP_CACHE_DIR environment variable"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "${HTTP_CACHE_PORT}" ]
|
||||
then
|
||||
HTTP_CACHE_PORT=8080
|
||||
fi
|
||||
|
||||
until ! lsof -Pi :$HTTP_CACHE_PORT -sTCP:LISTEN -t >/dev/null
|
||||
do
|
||||
echo "Port ${HTTP_CACHE_PORT} already in use, trying another..."
|
||||
((HTTP_CACHE_PORT=HTTP_CACHE_PORT+1))
|
||||
sleep 1
|
||||
done
|
||||
|
||||
if [ -z "${LOGGING_DIR}" ]
|
||||
then
|
||||
echo "Must set LOGGING_DIR environment variable"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
python3 -m http.server $HTTP_CACHE_PORT --directory $HTTP_CACHE_DIR 2>&1 > ${LOGGING_DIR}/http.log &
|
||||
export HTTP_CACHE_PID=$!
|
||||
|
||||
export HTTP_CACHE_URL=http://localhost:$HTTP_CACHE_PORT
|
||||
export LAMMPS_HTTP_CACHE_CONFIG=$HTTP_CACHE_DIR/proxy.cmake
|
||||
echo "Running local HTTP cache server on $HTTP_CACHE_URL (pid: $HTTP_CACHE_PID)"
|
||||
|
||||
function deactivate_http_cache {
|
||||
echo "Shutting down HTTP cache server..."
|
||||
kill $HTTP_CACHE_PID
|
||||
unset HTTP_CACHE_PID
|
||||
unset LAMMPS_HTTP_CACHE_CONFIG
|
||||
}
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
#!/bin/bash
|
||||
# needs to be sourced
|
||||
|
||||
if [ -z "${PIP_CACHE_DIR}" ]
|
||||
then
|
||||
echo "Must set PIP_CACHE_DIR environment variable"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
export PIP_NO_INDEX=1
|
||||
export PIP_FIND_LINKS=$PIP_CACHE_DIR
|
||||
|
||||
echo "Disabling pip index and use local cache directory..."
|
||||
|
||||
function deactivate_pip_cache {
|
||||
echo "Removing local pip cache configuration..."
|
||||
unset PIP_NO_INDEX
|
||||
unset PIP_FIND_LINKS
|
||||
}
|
||||
|
|
@ -0,0 +1,79 @@
|
|||
#!/bin/bash
|
||||
|
||||
if [ -z "${LAMMPS_CACHING_DIR}" ]
|
||||
then
|
||||
export LAMMPS_CACHING_DIR=$HOME/.cache/lammps
|
||||
echo "environment variable LAMMPS_CACHING_DIR not set"
|
||||
echo "Using default $LAMMPS_CACHING_DIR as cache directory..."
|
||||
else
|
||||
echo "Using $LAMMPS_CACHING_DIR as cache directory..."
|
||||
fi
|
||||
|
||||
if test -n "$BASH" ; then script=$BASH_SOURCE
|
||||
else script=$0
|
||||
fi
|
||||
|
||||
SCRIPT_DIR="$(dirname "$(realpath "$script")")"
|
||||
CACHE_SCRIPTS_DIR=${SCRIPT_DIR}/scripts
|
||||
|
||||
export GITHUB_PROXY_DIR=$LAMMPS_CACHING_DIR/github
|
||||
export LOGGING_DIR=$LAMMPS_CACHING_DIR/logs
|
||||
export PIP_CACHE_DIR=$LAMMPS_CACHING_DIR/pip
|
||||
export HTTP_CACHE_DIR=$LAMMPS_CACHING_DIR/http
|
||||
|
||||
if [ ! -d $GITHUB_PROXY_DIR ]
|
||||
then
|
||||
echo "GitHub proxy directory missing"
|
||||
return
|
||||
fi
|
||||
|
||||
if [ ! -d $LOGGING_DIR ]
|
||||
then
|
||||
echo "Logging directory missing"
|
||||
return
|
||||
fi
|
||||
|
||||
if [ ! -d $PIP_CACHE_DIR ]
|
||||
then
|
||||
echo "pip cache directory missing"
|
||||
return
|
||||
fi
|
||||
|
||||
if [ ! -d $HTTP_CACHE_DIR ]
|
||||
then
|
||||
echo "HTTP cache directory missing"
|
||||
return
|
||||
fi
|
||||
|
||||
echo "##############################################################################"
|
||||
echo "Setting up LAMMPS offline compilation environment"
|
||||
echo "##############################################################################"
|
||||
|
||||
source ${CACHE_SCRIPTS_DIR}/use_git_cache.sh
|
||||
source ${CACHE_SCRIPTS_DIR}/use_pip_cache.sh
|
||||
source ${CACHE_SCRIPTS_DIR}/use_http_cache.sh
|
||||
|
||||
echo "##############################################################################"
|
||||
echo
|
||||
echo "Prepend the following CMake options to your builds:"
|
||||
echo
|
||||
echo "-D LAMMPS_DOWNLOADS_URL=\${HTTP_CACHE_URL} -C \${LAMMPS_HTTP_CACHE_CONFIG}"
|
||||
echo
|
||||
echo "or"
|
||||
echo
|
||||
echo "-D LAMMPS_DOWNLOADS_URL=${HTTP_CACHE_URL} -C ${LAMMPS_HTTP_CACHE_CONFIG}"
|
||||
echo
|
||||
echo "pip installations and git clones (from git://) are automatically redirected"
|
||||
echo
|
||||
echo Use 'deactivate_caches' to revert changes
|
||||
echo
|
||||
echo "##############################################################################"
|
||||
|
||||
function deactivate_caches {
|
||||
deactivate_http_cache
|
||||
deactivate_pip_cache
|
||||
deactivate_git_cache
|
||||
unset -f deactivate_http_cache
|
||||
unset -f deactivate_pip_cache
|
||||
unset -f deactivate_git_cache
|
||||
}
|
||||
Loading…
Reference in New Issue