diff --git a/.gitattributes b/.gitattributes index 02b13c8ee5..c93225ca26 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,3 +1,4 @@ .gitattributes export-ignore .gitignore export-ignore .github export-ignore +.lgtm.yml export-ignore diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 8457af3af9..afdac828f8 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -13,29 +13,37 @@ lib/kim/* @ellio167 lib/mesont/* @iafoss # whole packages -src/COMPRESS/* @akohlmey +src/COMPRESS/* @rbberger src/GPU/* @ndtrung81 src/KOKKOS/* @stanmoore1 src/KIM/* @ellio167 src/LATTE/* @cnegre src/MESSAGE/* @sjplimp +src/MLIAP/* @athomps +src/SNAP/* @athomps src/SPIN/* @julient31 +src/USER-BROWNIAN/* @samueljmcameron src/USER-CGDNA/* @ohenrich src/USER-CGSDK/* @akohlmey src/USER-COLVARS/* @giacomofiorin +src/USER-FEP/* @agiliopadua +src/USER-HDNNP/* @singraber src/USER-INTEL/* @wmbrownintel src/USER-MANIFOLD/* @Pakketeretet2 +src/USER-MDI/* @taylor-a-barnes src/USER-MEAMC/* @martok src/USER-MESONT/* @iafoss src/USER-MOFFF/* @hheenen src/USER-MOLFILE/* @akohlmey src/USER-NETCDF/* @pastewka +src/USER-PACE/* @yury-lysogorskiy src/USER-PLUMED/* @gtribello src/USER-PHONON/* @lingtikong src/USER-PTM/* @pmla src/USER-OMP/* @akohlmey src/USER-QMMM/* @akohlmey src/USER-REAXC/* @hasanmetin +src/USER-REACTION/* @jrgissing src/USER-SCAFACOS/* @rhalver src/USER-TALLY/* @akohlmey src/USER-UEF/* @danicholson @@ -47,7 +55,6 @@ src/GPU/pair_vashishta_gpu.* @andeplane src/KOKKOS/pair_vashishta_kokkos.* @andeplane src/MANYBODY/pair_vashishta_table.* @andeplane src/MANYBODY/pair_atm.* @sergeylishchuk -src/USER-REACTION/fix_bond_react.* @jrgissing src/USER-MISC/*_grem.* @dstelter92 src/USER-MISC/compute_stress_mop*.* @RomainVermorel @@ -122,6 +129,8 @@ tools/emacs/* @HaoZeke tools/singularity/* @akohlmey @rbberger tools/code_standard/* @rbberger tools/valgrind/* @akohlmey +tools/swig/* @akohlmey +tools/offline/* @rbberger # tests unittest/* @akohlmey @rbberger @@ -130,7 +139,7 @@ unittest/* @akohlmey @rbberger cmake/* @junghans @rbberger cmake/Modules/Packages/USER-COLVARS.cmake @junghans @rbberger @giacomofiorin cmake/Modules/Packages/KIM.cmake @junghans @rbberger @ellio167 -cmake/presets/*.cmake @junghans @rbberger @akohlmey +cmake/presets/*.cmake @akohlmey # python python/* @rbberger @@ -142,6 +151,7 @@ fortran/* @akohlmey doc/utils/*/* @rbberger doc/Makefile @rbberger doc/README @rbberger +examples/plugin/* @akohlmey # for releases src/version.h @sjplimp diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 62e7186360..31b9becc0c 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -26,11 +26,11 @@ __ ## I don't want to read this whole thing I just have a question! -> **Note:** Please do not file an issue to ask a general question about LAMMPS, its features, how to use specific commands, or how perform simulations or analysis in LAMMPS. Instead post your question to the ['lammps-users' mailing list](https://lammps.sandia.gov/mail.html). You do not need to be subscribed to post to the list (but a mailing list subscription avoids having your post delayed until it is approved by a mailing list moderator). Most posts to the mailing list receive a response within less than 24 hours. Before posting to the mailing list, please read the [mailing list guidelines](https://lammps.sandia.gov/guidelines.html). Following those guidelines will help greatly to get a helpful response. Always mention which LAMMPS version you are using. +> **Note:** Please do not file an issue to ask a general question about LAMMPS, its features, how to use specific commands, or how perform simulations or analysis in LAMMPS. Instead post your question to either the ['lammps-users' mailing list](https://lammps.sandia.gov/mail.html) or the [LAMMPS Material Science Discourse forum](https://matsci.org/lammps). You do not need to be subscribed to post to the list (but a mailing list subscription avoids having your post delayed until it is approved by a mailing list moderator). Most posts to the mailing list receive a response within less than 24 hours. Before posting to the mailing list, please read the [mailing list guidelines](https://lammps.sandia.gov/guidelines.html). Following those guidelines will help greatly to get a helpful response. Always mention which LAMMPS version you are using. The LAMMPS forum was recently created as part of a larger effort to build a materials science community and have discussions not just about using LAMMPS. Thus the forum may be also used for discussions that would be off-topic for the mailing list. Those will just have to be moved to a more general category. ## How Can I Contribute? -There are several ways how you can actively contribute to the LAMMPS project: you can discuss compiling and using LAMMPS, and solving LAMMPS related problems with other LAMMPS users on the lammps-users mailing list, you can report bugs or suggest enhancements by creating issues on GitHub (or posting them to the lammps-users mailing list), and you can contribute by submitting pull requests on GitHub or e-mail your code +There are several ways how you can actively contribute to the LAMMPS project: you can discuss compiling and using LAMMPS, and solving LAMMPS related problems with other LAMMPS users on the lammps-users mailing list, you can report bugs or suggest enhancements by creating issues on GitHub (or posting them to the lammps-users mailing list or posting in the LAMMPS Materials Science Discourse forum), and you can contribute by submitting pull requests on GitHub or e-mail your code to one of the [LAMMPS core developers](https://lammps.sandia.gov/authors.html). As you may see from the aforementioned developer page, the LAMMPS software package includes the efforts of a very large number of contributors beyond the principal authors and maintainers. ### Discussing How To Use LAMMPS @@ -42,6 +42,8 @@ Anyone can browse/search previous questions/answers in the archives. You do not If you post a message and you are a subscriber, your message will appear immediately. If you are not a subscriber, your message will be moderated, which typically takes one business day. Either way, when someone replies the reply will usually be sent to both, your personal email address and the mailing list. When replying to people, that responded to your post to the list, please always included the mailing list in your replies (i.e. use "Reply All" and **not** "Reply"). Responses will appear on the list in a few minutes, but it can take a few hours for postings and replies to show up in the SourceForge archive. Sending replies also to the mailing list is important, so that responses are archived and people with a similar issue can search for possible solutions in the mailing list archive. +The LAMMPS Materials Science Discourse forum was created recently to facilitate discussion not just about LAMMPS and as part of a larger effort towards building a materials science community. The forum contains a read-only sub-category with the continually updated mailing list archive, so you won't miss anything by joining only the forum and not the mailing list. + ### Reporting Bugs While developers writing code for LAMMPS are careful to test their code, LAMMPS is such a large and complex software, that it is impossible to test for all combinations of features under all normal and not so normal circumstances. Thus bugs do happen, and if you suspect, that you have encountered one, please try to document it and report it as an [Issue](https://github.com/lammps/lammps/issues) on the LAMMPS GitHub project web page. However, before reporting a bug, you need to check whether this is something that may have already been corrected. The [Latest Features and Bug Fixes in LAMMPS](https://lammps.sandia.gov/bug.html) web page lists all significant changes to LAMMPS over the years. It also tells you what the current latest development version of LAMMPS is, and you should test whether your issue still applies to that version. diff --git a/.github/codeql/cpp.yml b/.github/codeql/cpp.yml new file mode 100644 index 0000000000..6244ba707d --- /dev/null +++ b/.github/codeql/cpp.yml @@ -0,0 +1,4 @@ +paths: + - src + - lib + - tools diff --git a/.github/codeql/python.yml b/.github/codeql/python.yml new file mode 100644 index 0000000000..c02a54e3ea --- /dev/null +++ b/.github/codeql/python.yml @@ -0,0 +1,5 @@ +paths: + - python/lammps + +queries: + - uses: security-and-quality diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 827306c9aa..064ba6c652 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -31,16 +31,18 @@ jobs: uses: github/codeql-action/init@v1 with: languages: ${{ matrix.language }} + config-file: ./.github/codeql/${{ matrix.language }}.yml - name: Create Build Environment - run: cmake -E make_directory ${{github.workspace}}/build + if: ${{ matrix.language == 'cpp' }} + run: mkdir build - name: Building LAMMPS via CMake if: ${{ matrix.language == 'cpp' }} shell: bash - working-directory: ${{github.workspace}}/build + working-directory: build run: | - cmake -C $GITHUB_WORKSPACE/cmake/presets/most.cmake $GITHUB_WORKSPACE/cmake + cmake -C ../cmake/presets/most.cmake ../cmake cmake --build . --parallel 2 - name: Perform CodeQL Analysis diff --git a/.github/workflows/unittest-macos.yml b/.github/workflows/unittest-macos.yml index f62b3046c9..88e28513e8 100644 --- a/.github/workflows/unittest-macos.yml +++ b/.github/workflows/unittest-macos.yml @@ -10,6 +10,8 @@ jobs: name: MacOS Unit Test if: ${{ github.repository == 'lammps/lammps' }} runs-on: macos-latest + env: + CCACHE_DIR: ${{ github.workspace }}/.ccache steps: - name: Checkout repository @@ -17,20 +19,36 @@ jobs: with: fetch-depth: 2 + - name: Install ccache + run: brew install ccache + - name: Create Build Environment - run: cmake -E make_directory ${{github.workspace}}/build + run: mkdir build + + - name: Set up ccache + uses: actions/cache@v2 + with: + path: ${{ env.CCACHE_DIR }} + key: macos-ccache-${{ github.sha }} + restore-keys: macos-ccache- - name: Building LAMMPS via CMake shell: bash - working-directory: ${{github.workspace}}/build + working-directory: build run: | - cmake -C $GITHUB_WORKSPACE/cmake/presets/clang.cmake \ - -C $GITHUB_WORKSPACE/cmake/presets/most.cmake \ - $GITHUB_WORKSPACE/cmake \ - -DENABLE_TESTING=ON -DBUILD_SHARED_LIBS=ON -DLAMMPS_EXCEPTIONS=ON + ccache -z + cmake -C ../cmake/presets/clang.cmake \ + -C ../cmake/presets/most.cmake \ + -D CMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -D CMAKE_C_COMPILER_LAUNCHER=ccache \ + -D ENABLE_TESTING=on \ + -D BUILD_SHARED_LIBS=on \ + -D LAMMPS_EXCEPTIONS=on \ + ../cmake cmake --build . --parallel 2 + ccache -s - name: Run Tests - working-directory: ${{github.workspace}}/build + working-directory: build shell: bash run: ctest -V diff --git a/.lgtm.yml b/.lgtm.yml new file mode 100644 index 0000000000..2f2a31f292 --- /dev/null +++ b/.lgtm.yml @@ -0,0 +1,14 @@ +extraction: + cpp: + configure: + command: + - "mkdir build" + - "cd build" + - "cmake -G Ninja -C ../cmake/presets/minimal.cmake ../cmake" + index: + build_command: + - "cd build" + - "ninja" + python: + python_setup: + version: 3 diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000000..9f65e2f88e --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,35 @@ +# Security Policy + +LAMMPS is designed as a user-level application to conduct computer +simulations for research using classical mechanics. As such LAMMPS +depends to some degrees on users providing correctly formatted input and +LAMMPS needs to read and write files based on uncontrolled user input. +As a parallel application for use in high-performance computing +environments, performance critical steps are also done without checking +data. + +LAMMPS also is interfaced to a number of external libraries, including +libraries with experimental research software, that are not validated +and tested by the LAMMPS developers, so it is easy to import bad +behavior from calling functions in one of those libraries. + +Thus is is quite easy to crash LAMMPS through malicious input and do all +kinds of filesystem manipulations. And because of that LAMMPS should +**NEVER** be compiled or **run** as superuser, either from a "root" or +"administrator" account directly or indirectly via "sudo" or "su". + +Therefore what could be seen as a security vulnerability is usually +either a user mistake or a bug in the code. Bugs can be reported in +the LAMMPS project +[issue tracker on GitHub](https://github.com/lammps/lammps/issues). + + +# Version Updates + +LAMMPS follows continuous release development model. We aim to keep all +release versions (stable or patch) fully functional and employ a variety +of automatic testing procedures to detect failures of existing +functionality from adding new features before releases are made. Thus +bugfixes and updates are only integrated into the current development +branch and thus the next (patch) release and users are recommended to +update regularly. diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 426d2f76e0..2b791f29a4 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -7,6 +7,11 @@ cmake_minimum_required(VERSION 3.10) if(POLICY CMP0074) cmake_policy(SET CMP0074 NEW) endif() +# set policy to silence warnings about missing executable permissions in +# pythonx.y-config when cross-compiling. review occasionally if it may be set to NEW +if (POLICY CMP0109) + cmake_policy(SET CMP0109 OLD) +endif() ######################################## project(lammps CXX) @@ -104,6 +109,12 @@ endif() option(BUILD_TOOLS "Build and install LAMMPS tools (msi2lmp, binary2txt, chain)" OFF) option(BUILD_LAMMPS_SHELL "Build and install the LAMMPS shell" OFF) +# allow enabling clang-tidy for C++ files +set(ENABLE_CLANG_TIDY OFF CACHE BOOL "Include clang-tidy processing when compiling") +if(ENABLE_CLANG_TIDY) + set(CMAKE_CXX_CLANG_TIDY "clang-tidy;-checks=*-header-filter=.*" CACHE STRING "") +endif() + include(GNUInstallDirs) file(GLOB ALL_SOURCES ${LAMMPS_SOURCE_DIR}/[^.]*.cpp) file(GLOB MAIN_SOURCES ${LAMMPS_SOURCE_DIR}/main.cpp) @@ -121,10 +132,10 @@ set(STANDARD_PACKAGES ASPHERE BODY CLASS2 COLLOID COMPRESS DIPOLE PLUGIN QEQ REPLICA RIGID SHOCK SPIN SNAP SRD KIM PYTHON MSCG MPIIO VORONOI USER-ADIOS USER-ATC USER-AWPMD USER-BOCS USER-CGDNA USER-MESODPD USER-CGSDK USER-COLVARS USER-DIFFRACTION USER-DPD USER-DRUDE USER-EFF USER-FEP USER-H5MD - USER-LB USER-MANIFOLD USER-MEAMC USER-MESONT USER-MGPT USER-MISC USER-MOFFF + USER-LB USER-MANIFOLD USER-MDI USER-MEAMC USER-MESONT USER-MGPT USER-MISC USER-MOFFF USER-MOLFILE USER-NETCDF USER-PHONON USER-PLUMED USER-PTM USER-QTB USER-RANN USER-REACTION USER-REAXC USER-SCAFACOS USER-SDPD USER-SMD USER-SMTBQ USER-SPH - USER-TALLY USER-UEF USER-VTK USER-QUIP USER-QMMM USER-YAFF USER-PACE) + USER-TALLY USER-UEF USER-VTK USER-QUIP USER-QMMM USER-YAFF USER-PACE USER-BROWNIAN) set(SUFFIX_PACKAGES CORESHELL GPU KOKKOS OPT USER-INTEL USER-OMP) @@ -245,69 +256,6 @@ if(BUILD_OMP) target_link_libraries(lammps PRIVATE OpenMP::OpenMP_CXX) endif() -# Compiler specific features for testing -if(${CMAKE_CXX_COMPILER_ID} STREQUAL "GNU") - option(ENABLE_COVERAGE "Enable collecting code coverage data" OFF) - mark_as_advanced(ENABLE_COVERAGE) - if(ENABLE_COVERAGE) - if(CMAKE_VERSION VERSION_LESS 3.13) - if(CMAKE_CXX_FLAGS) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --coverage") - else() - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_${CMAKE_BUILD_TYPE}_FLAGS} --coverage") - endif() - else() - target_compile_options(lammps PUBLIC --coverage) - target_link_options(lammps PUBLIC --coverage) - endif() - endif() -endif() - -####################################### -# add custom target for IWYU analysis -####################################### -set(ENABLE_IWYU OFF CACHE BOOL "Add 'iwyu' build target to call the include-what-you-use tool") -mark_as_advanced(ENABLE_IWYU) -if(ENABLE_IWYU) - set(CMAKE_EXPORT_COMPILE_COMMANDS ON) - find_program(IWYU_EXE NAMES include-what-you-use iwyu) - find_program(IWYU_TOOL NAMES iwyu_tool iwyu-tool iwyu_tool.py) - if (IWYU_EXE AND IWYU_TOOL) - add_custom_target( - iwyu - ${IWYU_TOOL} -o clang -p ${CMAKE_CURRENT_BINARY_DIR} -- -Xiwyu --mapping_file=${CMAKE_CURRENT_SOURCE_DIR}/iwyu/iwyu-extra-map.imp - COMMENT "Running IWYU") - add_dependencies(iwyu lammps) - else() - message(FATAL_ERROR "To use IWYU you need the include-what-you-use/iwyu executable" - "and the iwyu-tool/iwyu_tool script installed in your PATH") - endif() -endif() - -set(ENABLE_SANITIZER "none" CACHE STRING "Select a code sanitizer option (none (default), address, leak, thread, undefined)") -mark_as_advanced(ENABLE_SANITIZER) -set(ENABLE_SANITIZER_VALUES none address leak thread undefined) -set_property(CACHE ENABLE_SANITIZER PROPERTY STRINGS ${ENABLE_SANITIZER_VALUES}) -validate_option(ENABLE_SANITIZER ENABLE_SANITIZER_VALUES) -string(TOLOWER ${ENABLE_SANITIZER} ENABLE_SANITIZER) -if(NOT ENABLE_SANITIZER STREQUAL "none") - if((${CMAKE_CXX_COMPILER_ID} STREQUAL "GNU") OR (${CMAKE_CXX_COMPILER_ID} STREQUAL "Clang")) - if(CMAKE_VERSION VERSION_LESS 3.13) - if(CMAKE_CXX_FLAGS) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=${ENABLE_SANITIZER}") - else() - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_${CMAKE_BUILD_TYPE}_FLAGS} -fsanitize=${ENABLE_SANITIZER}") - endif() - else() - target_compile_options(lammps PUBLIC -fsanitize=${ENABLE_SANITIZER}) - target_link_options(lammps PUBLIC -fsanitize=${ENABLE_SANITIZER}) - endif() - else() - message(WARNING "ENABLE_SANITIZER option not supported by ${CMAKE_CXX_COMPILER_ID} compilers. Ignoring.") - set(ENABLE_SANITIZER "none") - endif() -endif() - if(PKG_MSCG OR PKG_USER-ATC OR PKG_USER-AWPMD OR PKG_USER-QUIP OR PKG_LATTE) enable_language(C) find_package(LAPACK) @@ -387,8 +335,8 @@ else() set(CUDA_REQUEST_PIC) endif() -foreach(PKG_WITH_INCL KSPACE PYTHON MLIAP VORONOI USER-COLVARS USER-MOLFILE USER-NETCDF USER-PLUMED USER-QMMM - USER-QUIP USER-SCAFACOS USER-SMD USER-VTK KIM LATTE MESSAGE MSCG COMPRESS USER-PACE) +foreach(PKG_WITH_INCL KSPACE PYTHON MLIAP VORONOI USER-COLVARS USER-MDI USER-MOLFILE USER-NETCDF USER-PLUMED + USER-QMMM USER-QUIP USER-SCAFACOS USER-SMD USER-VTK KIM LATTE MESSAGE MSCG COMPRESS USER-PACE) if(PKG_${PKG_WITH_INCL}) include(Packages/${PKG_WITH_INCL}) endif() diff --git a/cmake/Modules/Documentation.cmake b/cmake/Modules/Documentation.cmake index c0028676b7..16dd5dd4fa 100644 --- a/cmake/Modules/Documentation.cmake +++ b/cmake/Modules/Documentation.cmake @@ -55,8 +55,8 @@ if(BUILD_DOC) COMMAND ${DOCENV_BINARY_DIR}/pip $ENV{PIP_OPTIONS} install -r ${DOC_BUILD_DIR}/requirements.txt --upgrade ) - set(MATHJAX_URL "https://github.com/mathjax/MathJax/archive/3.1.2.tar.gz" CACHE STRING "URL for MathJax tarball") - set(MATHJAX_MD5 "a4a6a093a89bc2ccab1452d766b98e53" CACHE STRING "MD5 checksum of MathJax tarball") + set(MATHJAX_URL "https://github.com/mathjax/MathJax/archive/3.1.3.tar.gz" CACHE STRING "URL for MathJax tarball") + set(MATHJAX_MD5 "d1c98c746888bfd52ca8ebc10704f92f" CACHE STRING "MD5 checksum of MathJax tarball") mark_as_advanced(MATHJAX_URL) # download mathjax distribution and unpack to folder "mathjax" diff --git a/cmake/Modules/GenerateOpenCLHeader.cmake b/cmake/Modules/GenerateOpenCLHeader.cmake new file mode 100644 index 0000000000..18dfac1604 --- /dev/null +++ b/cmake/Modules/GenerateOpenCLHeader.cmake @@ -0,0 +1,3 @@ +# utility script to call WriteOpenCLHeader function +include(${SOURCE_DIR}/Modules/OpenCLUtils.cmake) +WriteOpenCLHeader(${VARNAME} ${HEADER_FILE} ${SOURCE_FILES}) diff --git a/cmake/Modules/OpenCLLoader.cmake b/cmake/Modules/OpenCLLoader.cmake index 54eaab4795..ea83f8e447 100644 --- a/cmake/Modules/OpenCLLoader.cmake +++ b/cmake/Modules/OpenCLLoader.cmake @@ -1,6 +1,6 @@ message(STATUS "Downloading and building OpenCL loader library") -set(OPENCL_LOADER_URL "${LAMMPS_THIRDPARTY_URL}/opencl-loader-2020.12.18.tar.gz" CACHE STRING "URL for OpenCL loader tarball") -set(OPENCL_LOADER_MD5 "011cdcbd41030be94f3fced6d763a52a" CACHE STRING "MD5 checksum of OpenCL loader tarball") +set(OPENCL_LOADER_URL "${LAMMPS_THIRDPARTY_URL}/opencl-loader-2021.05.02.tar.gz" CACHE STRING "URL for OpenCL loader tarball") +set(OPENCL_LOADER_MD5 "29180b05056578afda92f0d394c3a373" CACHE STRING "MD5 checksum of OpenCL loader tarball") mark_as_advanced(OPENCL_LOADER_URL) mark_as_advanced(OPENCL_LOADER_MD5) diff --git a/cmake/Modules/OpenCLUtils.cmake b/cmake/Modules/OpenCLUtils.cmake index a9441ce518..eb17da0b3d 100644 --- a/cmake/Modules/OpenCLUtils.cmake +++ b/cmake/Modules/OpenCLUtils.cmake @@ -1,10 +1,8 @@ -function(GenerateOpenCLHeader varname outfile files) - message("Creating ${outfile}...") +function(WriteOpenCLHeader varname outfile files) file(WRITE ${outfile} "const char * ${varname} = \n") - math(EXPR ARG_END "${ARGC}-1") + separate_arguments(files) - foreach(IDX RANGE 2 ${ARG_END}) - list(GET ARGV ${IDX} filename) + foreach(filename ${files}) file(READ ${filename} content) string(REGEX REPLACE "\\s*//[^\n]*\n" "\n" content "${content}") string(REGEX REPLACE "\\\\" "\\\\\\\\" content "${content}") @@ -15,4 +13,16 @@ function(GenerateOpenCLHeader varname outfile files) endforeach() file(APPEND ${outfile} ";\n") +endfunction(WriteOpenCLHeader) + +function(GenerateOpenCLHeader varname outfile files) + list(REMOVE_AT ARGV 0 1) + add_custom_command(OUTPUT ${outfile} + COMMAND ${CMAKE_COMMAND} -D SOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR} + -D VARNAME=${varname} + -D HEADER_FILE=${outfile} + -D SOURCE_FILES="${ARGV}" + -P ${CMAKE_CURRENT_SOURCE_DIR}/Modules/GenerateOpenCLHeader.cmake + DEPENDS ${ARGV} + COMMENT "Generating ${outfile}...") endfunction(GenerateOpenCLHeader) diff --git a/cmake/Modules/Packages/GPU.cmake b/cmake/Modules/Packages/GPU.cmake index b706537825..3b07246e80 100644 --- a/cmake/Modules/Packages/GPU.cmake +++ b/cmake/Modules/Packages/GPU.cmake @@ -112,10 +112,10 @@ if(GPU_API STREQUAL "CUDA") endif() cuda_compile_fatbin(GPU_GEN_OBJS ${GPU_LIB_CU} OPTIONS ${CUDA_REQUEST_PIC} - -DUNIX -O3 --use_fast_math -Wno-deprecated-gpu-targets -DNV_KERNEL -DUCL_CUDADR ${GPU_CUDA_GENCODE} -D_${GPU_PREC_SETTING}) + -DUNIX -O3 --use_fast_math -Wno-deprecated-gpu-targets -DNV_KERNEL -DUCL_CUDADR ${GPU_CUDA_GENCODE} -D_${GPU_PREC_SETTING} -DLAMMPS_${LAMMPS_SIZES}) cuda_compile(GPU_OBJS ${GPU_LIB_CUDPP_CU} OPTIONS ${CUDA_REQUEST_PIC} - -DUNIX -O3 --use_fast_math -Wno-deprecated-gpu-targets -DUCL_CUDADR ${GPU_CUDA_GENCODE} -D_${GPU_PREC_SETTING}) + -DUNIX -O3 --use_fast_math -Wno-deprecated-gpu-targets -DUCL_CUDADR ${GPU_CUDA_GENCODE} -D_${GPU_PREC_SETTING} -DLAMMPS_${LAMMPS_SIZES}) foreach(CU_OBJ ${GPU_GEN_OBJS}) get_filename_component(CU_NAME ${CU_OBJ} NAME_WE) @@ -145,7 +145,14 @@ if(GPU_API STREQUAL "CUDA") target_include_directories(nvc_get_devices PRIVATE ${CUDA_INCLUDE_DIRS}) elseif(GPU_API STREQUAL "OPENCL") - option(USE_STATIC_OPENCL_LOADER "Download and include a static OpenCL ICD loader" ON) + # the static OpenCL loader doesn't seem to work on macOS. use the system provided + # version by default instead (for as long as it will be available) + if("${CMAKE_SYSTEM_NAME}" STREQUAL "Darwin") + set(_opencl_static_default OFF) + else() + set(_opencl_static_default ON) + endif() + option(USE_STATIC_OPENCL_LOADER "Download and include a static OpenCL ICD loader" ${_opencl_static_default}) mark_as_advanced(USE_STATIC_OPENCL_LOADER) if (USE_STATIC_OPENCL_LOADER) include(OpenCLLoader) @@ -289,18 +296,18 @@ elseif(GPU_API STREQUAL "HIP") if(HIP_COMPILER STREQUAL "clang") add_custom_command(OUTPUT ${CUBIN_FILE} - VERBATIM COMMAND ${HIP_HIPCC_EXECUTABLE} --genco --offload-arch=${HIP_ARCH} -O3 -ffast-math -DUSE_HIP -D_${GPU_PREC_SETTING} -I${LAMMPS_LIB_SOURCE_DIR}/gpu -o ${CUBIN_FILE} ${CU_CPP_FILE} + VERBATIM COMMAND ${HIP_HIPCC_EXECUTABLE} --genco --offload-arch=${HIP_ARCH} -O3 -ffast-math -DUSE_HIP -D_${GPU_PREC_SETTING} -DLAMMPS_${LAMMPS_SIZES} -I${LAMMPS_LIB_SOURCE_DIR}/gpu -o ${CUBIN_FILE} ${CU_CPP_FILE} DEPENDS ${CU_CPP_FILE} COMMENT "Generating ${CU_NAME}.cubin") else() add_custom_command(OUTPUT ${CUBIN_FILE} - VERBATIM COMMAND ${HIP_HIPCC_EXECUTABLE} --genco -t="${HIP_ARCH}" -f=\"-O3 -ffast-math -DUSE_HIP -D_${GPU_PREC_SETTING} -I${LAMMPS_LIB_SOURCE_DIR}/gpu\" -o ${CUBIN_FILE} ${CU_CPP_FILE} + VERBATIM COMMAND ${HIP_HIPCC_EXECUTABLE} --genco -t="${HIP_ARCH}" -f=\"-O3 -ffast-math -DUSE_HIP -D_${GPU_PREC_SETTING} -DLAMMPS_${LAMMPS_SIZES} -I${LAMMPS_LIB_SOURCE_DIR}/gpu\" -o ${CUBIN_FILE} ${CU_CPP_FILE} DEPENDS ${CU_CPP_FILE} COMMENT "Generating ${CU_NAME}.cubin") endif() elseif(HIP_PLATFORM STREQUAL "nvcc") add_custom_command(OUTPUT ${CUBIN_FILE} - VERBATIM COMMAND ${HIP_HIPCC_EXECUTABLE} --fatbin --use_fast_math -DUSE_HIP -D_${GPU_PREC_SETTING} ${HIP_CUDA_GENCODE} -I${LAMMPS_LIB_SOURCE_DIR}/gpu -o ${CUBIN_FILE} ${CU_FILE} + VERBATIM COMMAND ${HIP_HIPCC_EXECUTABLE} --fatbin --use_fast_math -DUSE_HIP -D_${GPU_PREC_SETTING} -DLAMMPS_${LAMMPS_SIZES} ${HIP_CUDA_GENCODE} -I${LAMMPS_LIB_SOURCE_DIR}/gpu -o ${CUBIN_FILE} ${CU_FILE} DEPENDS ${CU_FILE} COMMENT "Generating ${CU_NAME}.cubin") endif() diff --git a/cmake/Modules/Packages/KOKKOS.cmake b/cmake/Modules/Packages/KOKKOS.cmake index 1f00516e08..e0e32730ec 100644 --- a/cmake/Modules/Packages/KOKKOS.cmake +++ b/cmake/Modules/Packages/KOKKOS.cmake @@ -37,8 +37,8 @@ if(DOWNLOAD_KOKKOS) list(APPEND KOKKOS_LIB_BUILD_ARGS "-DCMAKE_CXX_EXTENSIONS=${CMAKE_CXX_EXTENSIONS}") list(APPEND KOKKOS_LIB_BUILD_ARGS "-DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}") include(ExternalProject) - set(KOKKOS_URL "https://github.com/kokkos/kokkos/archive/3.3.01.tar.gz" CACHE STRING "URL for KOKKOS tarball") - set(KOKKOS_MD5 "08201d1c7cf5bc458ce0f5b44a629d5a" CACHE STRING "MD5 checksum of KOKKOS tarball") + set(KOKKOS_URL "https://github.com/kokkos/kokkos/archive/3.4.00.tar.gz" CACHE STRING "URL for KOKKOS tarball") + set(KOKKOS_MD5 "c2fdcedb6953e6160c765366f6045abb" CACHE STRING "MD5 checksum of KOKKOS tarball") mark_as_advanced(KOKKOS_URL) mark_as_advanced(KOKKOS_MD5) ExternalProject_Add(kokkos_build @@ -58,7 +58,7 @@ if(DOWNLOAD_KOKKOS) target_link_libraries(lmp PRIVATE LAMMPS::KOKKOS) add_dependencies(LAMMPS::KOKKOS kokkos_build) elseif(EXTERNAL_KOKKOS) - find_package(Kokkos 3.3.01 REQUIRED CONFIG) + find_package(Kokkos 3.4.00 REQUIRED CONFIG) target_link_libraries(lammps PRIVATE Kokkos::kokkos) target_link_libraries(lmp PRIVATE Kokkos::kokkos) else() diff --git a/cmake/Modules/Packages/USER-INTEL.cmake b/cmake/Modules/Packages/USER-INTEL.cmake index 90ab6167a3..ecad135b70 100644 --- a/cmake/Modules/Packages/USER-INTEL.cmake +++ b/cmake/Modules/Packages/USER-INTEL.cmake @@ -79,9 +79,11 @@ if(INTEL_ARCH STREQUAL "KNL") else() if(CMAKE_CXX_COMPILER_ID STREQUAL "Intel") include(CheckCXXCompilerFlag) - foreach(_FLAG -O2 -fp-model fast=2 -no-prec-div -qoverride-limits -qopt-zmm-usage=high -qno-offload -fno-alias -ansi-alias -restrict) - check_cxx_compiler_flag("${_FLAG}" COMPILER_SUPPORTS${_FLAG}) - if(COMPILER_SUPPORTS${_FLAG}) + foreach(_FLAG -O2 "-fp-model fast=2" -no-prec-div -qoverride-limits -qopt-zmm-usage=high -qno-offload -fno-alias -ansi-alias -restrict) + string(REGEX REPLACE "[ =\"]" "" _FLAGX ${_FLAG}) + check_cxx_compiler_flag("${_FLAG}" COMPILER_SUPPORTS${_FLAGX}) + if(COMPILER_SUPPORTS${_FLAGX}) + separate_arguments(_FLAG UNIX_COMMAND "${_FLAG}") target_compile_options(lammps PRIVATE ${_FLAG}) endif() endforeach() diff --git a/cmake/Modules/Packages/USER-MDI.cmake b/cmake/Modules/Packages/USER-MDI.cmake new file mode 100644 index 0000000000..88d7b5e498 --- /dev/null +++ b/cmake/Modules/Packages/USER-MDI.cmake @@ -0,0 +1,118 @@ +find_package(mdi QUIET) +if(${mdi_FOUND}) + set(DOWNLOAD_MDI_DEFAULT OFF) +else() + set(DOWNLOAD_MDI_DEFAULT ON) +endif() +option(DOWNLOAD_MDI "Download and compile the MDI library instead of using an already installed one" ${DOWNLOAD_MDI_DEFAULT}) + +if(DOWNLOAD_MDI) + message(STATUS "MDI download requested - we will build our own") + set(MDI_URL "https://github.com/MolSSI-MDI/MDI_Library/archive/v1.2.9.tar.gz" CACHE STRING "URL for MDI tarball") + set(MDI_MD5 "ddfa46d6ee15b4e59cfd527ec7212184" CACHE STRING "MD5 checksum for MDI tarball") + mark_as_advanced(MDI_URL) + mark_as_advanced(MDI_MD5) + enable_language(C) + + # only ON/OFF are allowed for "mpi" flag when building MDI library + # so translate boolean value of BUILD_MPI + # always disable MPI when cross-compiling to Windows. + if((BUILD_MPI) AND NOT((CMAKE_SYSTEM_NAME STREQUAL Windows) AND CMAKE_CROSSCOMPILING)) + set(MDI_USE_MPI ON) + else() + set(MDI_USE_MPI OFF) + endif() + + # detect if we have python development support and thus can enable python plugins + set(MDI_USE_PYTHON_PLUGINS OFF) + if(CMAKE_VERSION VERSION_LESS 3.12) + find_package(PythonLibs QUIET) # Deprecated since version 3.12 + if (PYTHONLIBS_FOUND) + set(MDI_USE_PYTHON_PLUGINS ON) + endif() + else() + find_package(Python QUIET COMPONENTS Development) + if (Python_Development_FOUND) + set(MDI_USE_PYTHON_PLUGINS ON) + endif() + endif() + + # download/ build MDI library + # always build static library with -fpic + # support cross-compilation and ninja-build + include(ExternalProject) + ExternalProject_Add(mdi_build + URL ${MDI_URL} + URL_MD5 ${MDI_MD5} + CMAKE_ARGS ${CMAKE_REQUEST_PIC} + -DCMAKE_INSTALL_PREFIX= + -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -DCMAKE_MAKE_PROGRAM=${CMAKE_MAKE_PROGRAM} + -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} + -Dlanguage=C + -Dlibtype=STATIC + -Dmpi=${MDI_USE_MPI} + -Dpython_plugins=${MDI_USE_PYTHON_PLUGINS} + UPDATE_COMMAND "" + INSTALL_COMMAND "" + BUILD_BYPRODUCTS "/MDI_Library/libmdi.a" + ) + + # where is the compiled library? + ExternalProject_get_property(mdi_build BINARY_DIR) + set(MDI_BINARY_DIR "${BINARY_DIR}/MDI_Library") + # workaround for older CMake versions + file(MAKE_DIRECTORY ${MDI_BINARY_DIR}) + + # create imported target for the MDI library + add_library(LAMMPS::MDI UNKNOWN IMPORTED) + add_dependencies(LAMMPS::MDI mdi_build) + set_target_properties(LAMMPS::MDI PROPERTIES + IMPORTED_LOCATION "${MDI_BINARY_DIR}/libmdi.a" + INTERFACE_INCLUDE_DIRECTORIES ${MDI_BINARY_DIR} + ) + + set(MDI_DEP_LIBS "") + # if compiling with python plugins we need + # to add python libraries as dependency. + if(MDI_USE_PYTHON_PLUGINS) + if(CMAKE_VERSION VERSION_LESS 3.12) + list(APPEND MDI_DEP_LIBS ${PYTHON_LIBRARIES}) + else() + list(APPEND MDI_DEP_LIBS Python::Python) + endif() + + endif() + # need to add support for dlopen/dlsym, except when compiling for Windows. + if(NOT ${CMAKE_SYSTEM_NAME} STREQUAL "Windows") + list(APPEND MDI_DEP_LIBS "${CMAKE_DL_LIBS}") + endif() + if(MDI_DEP_LIBS) + set_target_properties(LAMMPS::MDI PROPERTIES + IMPORTED_LINK_INTERFACE_LIBRARIES "${MDI_DEP_LIBS}") + endif() + + target_link_libraries(lammps PRIVATE LAMMPS::MDI) + target_link_libraries(lmp PRIVATE LAMMPS::MDI) + +else() + + find_package(mdi) + if(NOT mdi_FOUND) + message(FATAL_ERROR "MDI library not found. Help CMake to find it " + "by setting mdi_LIBRARY and mdi_INCLUDE_DIR, or set DOWNLOAD_MDI=ON " + "to download and compile it") + endif() + + # Link the lammps library against MDI + target_include_directories(lammps PRIVATE ${mdi_INCLUDE_DIR}) + target_link_libraries(lammps PRIVATE ${mdi_LIBRARY}) + + # Link the lammps executable against MDI + target_include_directories(lmp PRIVATE ${mdi_INCLUDE_DIR}) + target_link_libraries(lmp PRIVATE ${mdi_LIBRARY}) +endif() + +target_compile_definitions(lammps PRIVATE -DLMP_USER_MDI) +target_compile_definitions(lmp PRIVATE -DLMP_USER_MDI) diff --git a/cmake/Modules/Packages/USER-PACE.cmake b/cmake/Modules/Packages/USER-PACE.cmake index 66f228017c..ccc7423355 100644 --- a/cmake/Modules/Packages/USER-PACE.cmake +++ b/cmake/Modules/Packages/USER-PACE.cmake @@ -14,13 +14,12 @@ execute_process( WORKING_DIRECTORY ${CMAKE_BINARY_DIR} ) - file(GLOB PACE_EVALUATOR_INCLUDE_DIR ${CMAKE_BINARY_DIR}/lammps-user-pace-*/USER-PACE) file(GLOB PACE_EVALUATOR_SOURCES ${CMAKE_BINARY_DIR}/lammps-user-pace-*/USER-PACE/*.cpp) list(FILTER PACE_EVALUATOR_SOURCES EXCLUDE REGEX pair_pace.cpp) add_library(pace STATIC ${PACE_EVALUATOR_SOURCES}) -set_target_properties(pace PROPERTIES OUTPUT_NAME pace${LAMMPS_MACHINE}) +set_target_properties(pace PROPERTIES CXX_EXTENSIONS ON OUTPUT_NAME lammps_pace${LAMMPS_MACHINE}) target_include_directories(pace PUBLIC ${PACE_EVALUATOR_INCLUDE_DIR}) target_link_libraries(lammps PRIVATE pace) diff --git a/cmake/Modules/Testing.cmake b/cmake/Modules/Testing.cmake index 7fbd8212de..d1fa97a7a8 100644 --- a/cmake/Modules/Testing.cmake +++ b/cmake/Modules/Testing.cmake @@ -56,3 +56,76 @@ if(ENABLE_TESTING) get_filename_component(LAMMPS_UNITTEST_BIN ${CMAKE_BINARY_DIR}/unittest ABSOLUTE) add_subdirectory(${LAMMPS_UNITTEST_DIR} ${LAMMPS_UNITTEST_BIN}) endif() + +# Compiler specific features for testing +if(${CMAKE_CXX_COMPILER_ID} STREQUAL "GNU") + option(ENABLE_COVERAGE "Enable collecting code coverage data" OFF) + mark_as_advanced(ENABLE_COVERAGE) + if(ENABLE_COVERAGE) + if(CMAKE_VERSION VERSION_LESS 3.13) + if(CMAKE_CXX_FLAGS) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --coverage") + else() + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_${CMAKE_BUILD_TYPE}_FLAGS} --coverage") + endif() + else() + target_compile_options(lammps PUBLIC --coverage) + target_link_options(lammps PUBLIC --coverage) + endif() + endif() +endif() + +####################################### +# add custom target for IWYU analysis +####################################### +set(ENABLE_IWYU OFF CACHE BOOL "Add 'iwyu' build target to call the include-what-you-use tool") +mark_as_advanced(ENABLE_IWYU) +if(ENABLE_IWYU) + # enforce these settings + set(CMAKE_EXPORT_COMPILE_COMMANDS ON CACHE BOOL "Enable reporting compilation commands to compile_commands.json" FORCE) + if (NOT ((CMAKE_CXX_COMPILER_ID STREQUAL "Clang") OR (CMAKE_CXX_COMPILER_ID STREQUAL "GNU"))) + message(FATAL_ERROR "IWYU is only supported with Clang or GNU compilers") + endif() + # detect the "native" header folder so we can include them first + execute_process(COMMAND ${CMAKE_CXX_COMPILER} --print-search-dirs OUTPUT_VARIABLE IWYU_SEARCH_PATHS) + string(REGEX REPLACE ".*libraries: *=([^:]+):.*" "\\1/include" IWYU_EXTRA_INCLUDE_DIR ${IWYU_SEARCH_PATHS}) + find_program(IWYU_EXE NAMES include-what-you-use iwyu) + find_program(IWYU_TOOL NAMES iwyu_tool iwyu-tool iwyu_tool.py) + if (IWYU_EXE AND IWYU_TOOL) + add_custom_target( + iwyu + ${IWYU_TOOL} -o clang -p ${CMAKE_CURRENT_BINARY_DIR} -- -I${IWYU_EXTRA_INCLUDE_DIR} -Xiwyu --mapping_file=${CMAKE_CURRENT_SOURCE_DIR}/iwyu/iwyu-extra-map.imp + COMMENT "Running IWYU") + add_dependencies(iwyu lammps) + else() + message(FATAL_ERROR "To use IWYU you need the include-what-you-use/iwyu executable" + "and the iwyu-tool/iwyu_tool script installed in your PATH") + endif() +endif() + +####################################### +# select code sanitizer options +####################################### +set(ENABLE_SANITIZER "none" CACHE STRING "Select a code sanitizer option (none (default), address, leak, thread, undefined)") +mark_as_advanced(ENABLE_SANITIZER) +set(ENABLE_SANITIZER_VALUES none address leak thread undefined) +set_property(CACHE ENABLE_SANITIZER PROPERTY STRINGS ${ENABLE_SANITIZER_VALUES}) +validate_option(ENABLE_SANITIZER ENABLE_SANITIZER_VALUES) +string(TOLOWER ${ENABLE_SANITIZER} ENABLE_SANITIZER) +if(NOT ENABLE_SANITIZER STREQUAL "none") + if((${CMAKE_CXX_COMPILER_ID} STREQUAL "GNU") OR (${CMAKE_CXX_COMPILER_ID} STREQUAL "Clang")) + if(CMAKE_VERSION VERSION_LESS 3.13) + if(CMAKE_CXX_FLAGS) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=${ENABLE_SANITIZER}") + else() + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_${CMAKE_BUILD_TYPE}_FLAGS} -fsanitize=${ENABLE_SANITIZER}") + endif() + else() + target_compile_options(lammps PUBLIC -fsanitize=${ENABLE_SANITIZER}) + target_link_options(lammps PUBLIC -fsanitize=${ENABLE_SANITIZER}) + endif() + else() + message(WARNING "ENABLE_SANITIZER option not supported by ${CMAKE_CXX_COMPILER_ID} compilers. Ignoring.") + set(ENABLE_SANITIZER "none") + endif() +endif() diff --git a/cmake/Modules/YAML.cmake b/cmake/Modules/YAML.cmake index c50773568c..77ee804111 100644 --- a/cmake/Modules/YAML.cmake +++ b/cmake/Modules/YAML.cmake @@ -6,15 +6,27 @@ set(YAML_MD5 "bb15429d8fb787e7d3f1c83ae129a999" CACHE STRING "MD5 checksum of li mark_as_advanced(YAML_URL) mark_as_advanced(YAML_MD5) +# support cross-compilation to windows +if(CMAKE_CROSSCOMPILING AND (CMAKE_SYSTEM_NAME STREQUAL "Windows")) + if(CMAKE_SYSTEM_PROCESSOR STREQUAL "x86") + set(YAML_CROSS_HOST --host=i686-mingw64) + elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + set(YAML_CROSS_HOST --host=x86_64-mingw64) + else() + message(FATAL_ERROR "Unsupported cross-compilation " + " for ${CMAKE_SYSTEM_NAME}/${CMAKE_SYSTEM_PROCESSOR}" + " on ${CMAKE_HOST_SYSTEM}/${CMAKE_HOST_SYSTEM_PROCESSOR}") + endif() +endif() + ExternalProject_Add(libyaml URL ${YAML_URL} URL_MD5 ${YAML_MD5} SOURCE_DIR "${CMAKE_BINARY_DIR}/yaml-src" BINARY_DIR "${CMAKE_BINARY_DIR}/yaml-build" CONFIGURE_COMMAND /configure ${CONFIGURE_REQUEST_PIC} - CXX=${CMAKE_CXX_COMPILER} - CC=${CMAKE_C_COMPILER} - --prefix= --disable-shared + CXX=${CMAKE_CXX_COMPILER} CC=${CMAKE_C_COMPILER} + --prefix= --disable-shared ${YAML_CROSS_HOST} BUILD_BYPRODUCTS /lib/libyaml${CMAKE_STATIC_LIBRARY_SUFFIX} TEST_COMMAND "") diff --git a/cmake/Modules/generate_lmpgitversion.cmake b/cmake/Modules/generate_lmpgitversion.cmake index 4ff01c7501..b19716d74b 100644 --- a/cmake/Modules/generate_lmpgitversion.cmake +++ b/cmake/Modules/generate_lmpgitversion.cmake @@ -17,7 +17,7 @@ if(GIT_FOUND AND EXISTS ${LAMMPS_DIR}/.git) ERROR_QUIET WORKING_DIRECTORY ${LAMMPS_DIR} OUTPUT_STRIP_TRAILING_WHITESPACE) - execute_process(COMMAND ${GIT_EXECUTABLE} describe --dirty=-modified + execute_process(COMMAND ${GIT_EXECUTABLE} describe --dirty=-modified --always OUTPUT_VARIABLE temp_git_describe ERROR_QUIET WORKING_DIRECTORY ${LAMMPS_DIR} diff --git a/cmake/presets/all_off.cmake b/cmake/presets/all_off.cmake index bd94b9dbe5..7c0d1baeb7 100644 --- a/cmake/presets/all_off.cmake +++ b/cmake/presets/all_off.cmake @@ -2,16 +2,16 @@ # an existing package selection without losing any other settings set(ALL_PACKAGES ASPHERE BODY CLASS2 COLLOID COMPRESS CORESHELL DIPOLE GPU - GRANULAR KIM KOKKOS KSPACE LATTE MANYBODY MC MISC MESSAGE MLIAP - MOLECULE MPIIO MSCG OPT PERI POEMS PYTHON QEQ REPLICA RIGID SHOCK - SNAP SPIN SRD VORONOI - USER-ADIOS USER-ATC USER-AWPMD USER-BOCS USER-CGDNA USER-CGSDK - USER-COLVARS USER-DIFFRACTION USER-DPD USER-DRUDE USER-EFF USER-FEP - USER-H5MD USER-INTEL USER-LB USER-MANIFOLD USER-MEAMC USER-MESODPD - USER-MESONT USER-MGPT USER-MISC USER-MOFFF USER-MOLFILE USER-NETCDF USER-OMP - USER-PHONON USER-PLUMED USER-PTM USER-QMMM USER-QTB USER-QUIP - USER-REACTION USER-REAXC USER-SCAFACOS USER-SDPD USER-SMD USER-SMTBQ - USER-SPH USER-TALLY USER-UEF USER-VTK USER-YAFF) + GRANULAR KIM KOKKOS KSPACE LATTE MANYBODY MC MESSAGE MISC MLIAP MOLECULE + MPIIO MSCG OPT PERI PLUGIN POEMS PYTHON QEQ REPLICA RIGID SHOCK SNAP SPIN + SRD VORONOI + USER-ADIOS USER-ATC USER-AWPMD USER-BROWNIAN USER-BOCS USER-CGDNA USER-CGSDK + USER-COLVARS USER-DIFFRACTION USER-DPD USER-DRUDE USER-EFF USER-FEP USER-H5MD + USER-INTEL USER-LB USER-MANIFOLD USER-MDI USER-MEAMC USER-MESODPD USER-MESONT + USER-MGPT USER-MISC USER-MOFFF USER-MOLFILE USER-NETCDF USER-OMP USER-PACE + USER-PHONON USER-PLUMED USER-PTM USER-QMMM USER-QTB USER-QUIP USER-REACTION + USER-REAXC USER-SCAFACOS USER-SDPD USER-SMD USER-SMTBQ USER-SPH USER-TALLY + USER-UEF USER-VTK USER-YAFF) foreach(PKG ${ALL_PACKAGES}) set(PKG_${PKG} OFF CACHE BOOL "" FORCE) diff --git a/cmake/presets/all_on.cmake b/cmake/presets/all_on.cmake index 438c119c4c..5647c61833 100644 --- a/cmake/presets/all_on.cmake +++ b/cmake/presets/all_on.cmake @@ -4,16 +4,16 @@ # with just a working C++ compiler and an MPI library. set(ALL_PACKAGES ASPHERE BODY CLASS2 COLLOID COMPRESS CORESHELL DIPOLE GPU - GRANULAR KIM KOKKOS KSPACE LATTE MANYBODY MC MISC MESSAGE MLIAP - MOLECULE MPIIO MSCG OPT PERI POEMS PYTHON QEQ REPLICA RIGID SHOCK - SNAP SPIN SRD VORONOI - USER-ADIOS USER-ATC USER-AWPMD USER-BOCS USER-CGDNA USER-CGSDK - USER-COLVARS USER-DIFFRACTION USER-DPD USER-DRUDE USER-EFF USER-FEP - USER-H5MD USER-INTEL USER-LB USER-MANIFOLD USER-MEAMC USER-MESODPD - USER-MESONT USER-MGPT USER-MISC USER-MOFFF USER-MOLFILE USER-NETCDF USER-OMP - USER-PHONON USER-PLUMED USER-PTM USER-QMMM USER-QTB USER-QUIP - USER-REACTION USER-REAXC USER-SCAFACOS USER-SDPD USER-SMD USER-SMTBQ - USER-SPH USER-TALLY USER-UEF USER-VTK USER-YAFF) + GRANULAR KIM KOKKOS KSPACE LATTE MANYBODY MC MESSAGE MISC MLIAP MOLECULE + MPIIO MSCG OPT PERI PLUGIN POEMS PYTHON QEQ REPLICA RIGID SHOCK SNAP SPIN + SRD VORONOI + USER-ADIOS USER-ATC USER-AWPMD USER-BROWNIAN USER-BOCS USER-CGDNA USER-CGSDK + USER-COLVARS USER-DIFFRACTION USER-DPD USER-DRUDE USER-EFF USER-FEP USER-H5MD + USER-INTEL USER-LB USER-MANIFOLD USER-MDI USER-MEAMC USER-MESODPD USER-MESONT + USER-MGPT USER-MISC USER-MOFFF USER-MOLFILE USER-NETCDF USER-OMP USER-PACE + USER-PHONON USER-PLUMED USER-PTM USER-QMMM USER-QTB USER-QUIP USER-REACTION + USER-REAXC USER-SCAFACOS USER-SDPD USER-SMD USER-SMTBQ USER-SPH USER-TALLY + USER-UEF USER-VTK USER-YAFF) foreach(PKG ${ALL_PACKAGES}) set(PKG_${PKG} ON CACHE BOOL "" FORCE) diff --git a/cmake/presets/clang.cmake b/cmake/presets/clang.cmake index 47df6dcc6f..9ea077b14c 100644 --- a/cmake/presets/clang.cmake +++ b/cmake/presets/clang.cmake @@ -4,10 +4,18 @@ set(CMAKE_CXX_COMPILER "clang++" CACHE STRING "" FORCE) set(CMAKE_C_COMPILER "clang" CACHE STRING "" FORCE) set(CMAKE_CXX_FLAGS_DEBUG "-Wall -Wextra -g" CACHE STRING "" FORCE) set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-Wall -Wextra -g -O2 -DNDEBUG" CACHE STRING "" FORCE) +set(CMAKE_CXX_FLAGS_RELEASE "-O3 -DNDEBUG" CACHE STRING "" FORCE) +set(CMAKE_Fortran_FLAGS_DEBUG "-Wall -Wextra -g" CACHE STRING "" FORCE) +set(CMAKE_Fortran_FLAGS_RELWITHDEBINFO "-Wall -Wextra -g -O2 -DNDEBUG" CACHE STRING "" FORCE) +set(CMAKE_Fortran_FLAGS_RELEASE "-O3 -DNDEBUG" CACHE STRING "" FORCE) +set(CMAKE_C_FLAGS_DEBUG "-Wall -Wextra -g" CACHE STRING "" FORCE) +set(CMAKE_C_FLAGS_RELWITHDEBINFO "-Wall -Wextra -g -O2 -DNDEBUG" CACHE STRING "" FORCE) +set(CMAKE_C_FLAGS_RELEASE "-O3 -DNDEBUG" CACHE STRING "" FORCE) + set(MPI_CXX "clang++" CACHE STRING "" FORCE) set(MPI_CXX_COMPILER "mpicxx" CACHE STRING "" FORCE) -unset(HAVE_OMP_H_INCLUDE CACHE) +unset(HAVE_OMP_H_INCLUDE CACHE) set(OpenMP_C "clang" CACHE STRING "" FORCE) set(OpenMP_C_FLAGS "-fopenmp" CACHE STRING "" FORCE) set(OpenMP_C_LIB_NAMES "omp" CACHE STRING "" FORCE) diff --git a/cmake/presets/download.cmake b/cmake/presets/download.cmake index 84629ce785..ebab6ab658 100644 --- a/cmake/presets/download.cmake +++ b/cmake/presets/download.cmake @@ -1,7 +1,7 @@ -# preset that turns on packages with automatic downloads of sources of potentials -# compilation of libraries like Plumed or ScaFaCoS can take a considerable amount of time. +# Preset that turns on packages with automatic downloads of sources or potentials. +# Compilation of libraries like Plumed or ScaFaCoS can take a considerable amount of time. -set(ALL_PACKAGES KIM LATTE MSCG VORONOI USER-PLUMED USER-SCAFACOS USER-SMD USER-MESONT) +set(ALL_PACKAGES KIM LATTE MSCG VORONOI USER-PLUMED USER-SCAFACOS USER-SMD USER-MESONT USER-MDI USER-PACE) foreach(PKG ${ALL_PACKAGES}) set(PKG_${PKG} ON CACHE BOOL "" FORCE) @@ -9,9 +9,11 @@ endforeach() set(DOWNLOAD_KIM ON CACHE BOOL "" FORCE) set(DOWNLOAD_LATTE ON CACHE BOOL "" FORCE) +set(DOWNLOAD_MDI ON CACHE BOOL "" FORCE) set(DOWNLOAD_MSCG ON CACHE BOOL "" FORCE) set(DOWNLOAD_VORO ON CACHE BOOL "" FORCE) set(DOWNLOAD_EIGEN3 ON CACHE BOOL "" FORCE) +set(DOWNLOAD_PACE ON CACHE BOOL "" FORCE) set(DOWNLOAD_PLUMED ON CACHE BOOL "" FORCE) set(DOWNLOAD_SCAFACOS ON CACHE BOOL "" FORCE) diff --git a/cmake/presets/intel.cmake b/cmake/presets/intel.cmake index d101a0f699..9baf873608 100644 --- a/cmake/presets/intel.cmake +++ b/cmake/presets/intel.cmake @@ -3,10 +3,20 @@ set(CMAKE_CXX_COMPILER "icpc" CACHE STRING "" FORCE) set(CMAKE_C_COMPILER "icc" CACHE STRING "" FORCE) set(CMAKE_Fortran_COMPILER "ifort" CACHE STRING "" FORCE) +set(CMAKE_CXX_FLAGS_DEBUG "-Wall -Wextra -g" CACHE STRING "" FORCE) +set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-Wall -Wextra -g -O2 -DNDEBUG" CACHE STRING "" FORCE) +set(CMAKE_CXX_FLAGS_RELEASE "-O3 -DNDEBUG" CACHE STRING "" FORCE) +set(CMAKE_Fortran_FLAGS_DEBUG "-Wall -Wextra -g" CACHE STRING "" FORCE) +set(CMAKE_Fortran_FLAGS_RELWITHDEBINFO "-Wall -Wextra -g -O2 -DNDEBUG" CACHE STRING "" FORCE) +set(CMAKE_Fortran_FLAGS_RELEASE "-O3 -DNDEBUG" CACHE STRING "" FORCE) +set(CMAKE_C_FLAGS_DEBUG "-Wall -Wextra -g" CACHE STRING "" FORCE) +set(CMAKE_C_FLAGS_RELWITHDEBINFO "-Wall -Wextra -g -O2 -DNDEBUG" CACHE STRING "" FORCE) +set(CMAKE_C_FLAGS_RELEASE "-O3 -DNDEBUG" CACHE STRING "" FORCE) + set(MPI_CXX "icpc" CACHE STRING "" FORCE) set(MPI_CXX_COMPILER "mpicxx" CACHE STRING "" FORCE) -unset(HAVE_OMP_H_INCLUDE CACHE) +unset(HAVE_OMP_H_INCLUDE CACHE) set(OpenMP_C "icc" CACHE STRING "" FORCE) set(OpenMP_C_FLAGS "-qopenmp" CACHE STRING "" FORCE) set(OpenMP_C_LIB_NAMES "omp" CACHE STRING "" FORCE) diff --git a/cmake/presets/mingw-cross.cmake b/cmake/presets/mingw-cross.cmake index 32b17d43c2..e622211190 100644 --- a/cmake/presets/mingw-cross.cmake +++ b/cmake/presets/mingw-cross.cmake @@ -1,11 +1,11 @@ set(WIN_PACKAGES ASPHERE BODY CLASS2 COLLOID COMPRESS CORESHELL DIPOLE GPU GRANULAR KSPACE LATTE MANYBODY MC MISC MLIAP MOLECULE OPT PERI POEMS QEQ REPLICA RIGID SHOCK SNAP SPIN SRD VORONOI - USER-ATC USER-AWPMD USER-BOCS USER-CGDNA USER-CGSDK + USER-ATC USER-AWPMD USER-BOCS USER-BROWNIAN USER-CGDNA USER-CGSDK USER-COLVARS USER-DIFFRACTION USER-DPD USER-DRUDE USER-EFF USER-FEP USER-INTEL USER-MANIFOLD USER-MEAMC USER-MESODPD USER-MESONT USER-MISC USER-MGPT USER-MOFFF USER-MOLFILE USER-OMP - USER-PHONON USER-PTM USER-QTB USER-REACTION USER-REAXC + USER-PACE USER-PHONON USER-PTM USER-QTB USER-REACTION USER-REAXC USER-SDPD USER-SMD USER-SMTBQ USER-SPH USER-TALLY USER-UEF USER-YAFF) diff --git a/cmake/presets/most.cmake b/cmake/presets/most.cmake index 5dc58b735b..064b22595f 100644 --- a/cmake/presets/most.cmake +++ b/cmake/presets/most.cmake @@ -5,9 +5,9 @@ set(ALL_PACKAGES ASPHERE BODY CLASS2 COLLOID COMPRESS CORESHELL DIPOLE GRANULAR KSPACE MANYBODY MC MISC MLIAP MOLECULE OPT PERI PLUGIN POEMS PYTHON QEQ REPLICA RIGID SHOCK SNAP SPIN SRD VORONOI - USER-BOCS USER-CGDNA USER-CGSDK USER-COLVARS USER-DIFFRACTION - USER-DPD USER-DRUDE USER-EFF USER-FEP USER-MEAMC USER-MESODPD - USER-MISC USER-MOFFF USER-OMP USER-PHONON USER-REACTION + USER-BROWNIAN USER-BOCS USER-CGDNA USER-CGSDK USER-COLVARS + USER-DIFFRACTION USER-DPD USER-DRUDE USER-EFF USER-FEP USER-MEAMC + USER-MESODPD USER-MISC USER-MOFFF USER-OMP USER-PHONON USER-REACTION USER-REAXC USER-SDPD USER-SPH USER-SMD USER-UEF USER-YAFF) foreach(PKG ${ALL_PACKAGES}) diff --git a/cmake/presets/nolib.cmake b/cmake/presets/nolib.cmake index 0245b58cc7..17d6e09df3 100644 --- a/cmake/presets/nolib.cmake +++ b/cmake/presets/nolib.cmake @@ -1,10 +1,11 @@ # preset that turns off all packages that require some form of external # library or special compiler (fortran or cuda) or equivalent. -set(PACKAGES_WITH_LIB COMPRESS GPU KIM KOKKOS LATTE MPIIO MSCG PYTHON - VORONOI USER-ADIOS USER-ATC USER-AWPMD USER-H5MD USER-LB - USER-MOLFILE USER-MESONT USER-NETCDF USER-PLUMED USER-QMMM USER-QUIP - USER-SCAFACOS USER-SMD USER-VTK) +set(PACKAGES_WITH_LIB COMPRESS GPU KIM KOKKOS LATTE MESSAGE MPIIO MSCG + PYTHON VORONOI + USER-ADIOS USER-ATC USER-AWPMD USER-H5MD USER-LB USER-MOLFILE USER-MESONT + USER-MDI USER-NETCDF USER-PACE USER-PLUMED USER-QMMM USER-QUIP USER-SCAFACOS + USER-SMD USER-VTK) foreach(PKG ${PACKAGES_WITH_LIB}) set(PKG_${PKG} OFF CACHE BOOL "" FORCE) diff --git a/cmake/presets/oneapi.cmake b/cmake/presets/oneapi.cmake index ef04167305..403494c409 100644 --- a/cmake/presets/oneapi.cmake +++ b/cmake/presets/oneapi.cmake @@ -1,12 +1,22 @@ -# preset that will enable the LLVM based Intel compilers with support for MPI and OpenMP (on Linux boxes) +# preset that will enable the LLVM based Intel compilers with support for MPI and OpenMP and Fortran (on Linux boxes) set(CMAKE_CXX_COMPILER "icpx" CACHE STRING "" FORCE) set(CMAKE_C_COMPILER "icx" CACHE STRING "" FORCE) set(CMAKE_Fortran_COMPILER "ifx" CACHE STRING "" FORCE) +set(CMAKE_CXX_FLAGS_DEBUG "-Wall -Wextra -g" CACHE STRING "" FORCE) +set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-Wall -Wextra -g -O2 -DNDEBUG" CACHE STRING "" FORCE) +set(CMAKE_CXX_FLAGS_RELEASE "-O3 -DNDEBUG" CACHE STRING "" FORCE) +set(CMAKE_Fortran_FLAGS_DEBUG "-Wall -Wextra -g" CACHE STRING "" FORCE) +set(CMAKE_Fortran_FLAGS_RELWITHDEBINFO "-Wall -Wextra -g -O2 -DNDEBUG" CACHE STRING "" FORCE) +set(CMAKE_Fortran_FLAGS_RELEASE "-O3 -DNDEBUG" CACHE STRING "" FORCE) +set(CMAKE_C_FLAGS_DEBUG "-Wall -Wextra -g" CACHE STRING "" FORCE) +set(CMAKE_C_FLAGS_RELWITHDEBINFO "-Wall -Wextra -g -O2 -DNDEBUG" CACHE STRING "" FORCE) +set(CMAKE_C_FLAGS_RELEASE "-O3 -DNDEBUG" CACHE STRING "" FORCE) + set(MPI_CXX "icpx" CACHE STRING "" FORCE) set(MPI_CXX_COMPILER "mpicxx" CACHE STRING "" FORCE) -unset(HAVE_OMP_H_INCLUDE CACHE) +unset(HAVE_OMP_H_INCLUDE CACHE) set(OpenMP_C "icx" CACHE STRING "" FORCE) set(OpenMP_C_FLAGS "-qopenmp" CACHE STRING "" FORCE) set(OpenMP_C_LIB_NAMES "omp" CACHE STRING "" FORCE) diff --git a/doc/Makefile b/doc/Makefile index 7deaaf2a2e..e49d42ca77 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -236,7 +236,7 @@ $(VENV): ) $(MATHJAX): - @git clone --depth 1 git://github.com/mathjax/MathJax.git $@ + @git clone -b 3.1.4 -c advice.detachedHead=0 --depth 1 git://github.com/mathjax/MathJax.git $@ $(TXT2RST) $(ANCHORCHECK): $(VENV) @( \ diff --git a/doc/lammps.1 b/doc/lammps.1 index c10950643d..11438ac5c3 100644 --- a/doc/lammps.1 +++ b/doc/lammps.1 @@ -1,4 +1,4 @@ -.TH LAMMPS "8 April 2021" "2021-04-08" +.TH LAMMPS "14 May 2021" "2021-05-14" .SH NAME .B LAMMPS \- Molecular Dynamics Simulator. diff --git a/doc/src/Build_development.rst b/doc/src/Build_development.rst index 9c5f5ea61e..cc47a617ff 100644 --- a/doc/src/Build_development.rst +++ b/doc/src/Build_development.rst @@ -28,6 +28,28 @@ variable VERBOSE set to 1: ---------- +.. _clang-tidy + +Enable static code analysis with clang-tidy +------------------------------------------- + +The `clang-tidy tool `_ is a +static code analysis tool to diagnose (and potentially fix) typical +programming errors or coding style violations. It has a modular framework +of tests that can be adjusted to help identifying problems before they +become bugs and also assist in modernizing large code bases (like LAMMPS). +It can be enabled for all C++ code with the following CMake flag + +.. code-block:: bash + + -D ENABLE_CLANG_TIDY=value # value = no (default) or yes + +With this flag enabled all source files will be processed twice, first to +be compiled and then to be analyzed. Please note that the analysis can be +significantly more time consuming than the compilation itself. + +---------- + .. _iwyu_processing: Report missing and unneeded '#include' statements diff --git a/doc/src/Build_extras.rst b/doc/src/Build_extras.rst index d375d33e56..25ef36b430 100644 --- a/doc/src/Build_extras.rst +++ b/doc/src/Build_extras.rst @@ -49,6 +49,7 @@ This is the list of packages that may require additional steps. * :ref:`USER-COLVARS ` * :ref:`USER-H5MD ` * :ref:`USER-INTEL ` + * :ref:`USER-MDI ` * :ref:`USER-MESONT ` * :ref:`USER-MOLFILE ` * :ref:`USER-NETCDF ` @@ -467,6 +468,9 @@ They must be specified in uppercase. * - ARMV8_THUNDERX2 - HOST - ARMv8 Cavium ThunderX2 CPU + * - A64FX + - HOST + - ARMv8.2 with SVE Support * - WSM - HOST - Intel Westmere CPU (SSE 4.2) @@ -539,6 +543,9 @@ They must be specified in uppercase. * - AMPERE80 - GPU - NVIDIA Ampere generation CC 8.0 GPU + * - AMPERE86 + - GPU + - NVIDIA Ampere generation CC 8.6 GPU * - VEGA900 - GPU - AMD GPU MI25 GFX900 @@ -547,12 +554,12 @@ They must be specified in uppercase. - AMD GPU MI50/MI60 GFX906 * - VEGA908 - GPU - - AMD GPU GFX908 + - AMD GPU MI100 GFX908 * - INTEL_GEN - GPU - Intel GPUs Gen9+ -This list was last updated for version 3.3 of the Kokkos library. +This list was last updated for version 3.4 of the Kokkos library. .. tabs:: @@ -1533,6 +1540,35 @@ TBB and MKL. ---------- +.. _user-mdi: + +USER-MDI package +----------------------------- + +.. tabs:: + + .. tab:: CMake build + + .. code-block:: bash + + -D DOWNLOAD_MDI=value # download MDI Library for build, value = no (default) or yes + + .. tab:: Traditional make + + Before building LAMMPS, you must build the MDI Library in + ``lib/mdi``\ . You can do this by executing a command like one + of the following from the ``lib/mdi`` directory: + + .. code-block:: bash + + $ python Install.py -m gcc # build using gcc compiler + $ python Install.py -m icc # build using icc compiler + + The build should produce two files: ``lib/mdi/includelink/mdi.h`` + and ``lib/mdi/liblink/libmdi.so``\ . + +---------- + .. _user-mesont: USER-MESONT package diff --git a/doc/src/Commands_all.rst b/doc/src/Commands_all.rst index b43fd0ed56..c708228be7 100644 --- a/doc/src/Commands_all.rst +++ b/doc/src/Commands_all.rst @@ -67,6 +67,7 @@ An alphabetic list of all general LAMMPS commands. * :doc:`lattice ` * :doc:`log ` * :doc:`mass ` + * :doc:`mdi/engine ` * :doc:`message ` * :doc:`minimize ` * :doc:`min_modify ` diff --git a/doc/src/Commands_fix.rst b/doc/src/Commands_fix.rst index 671716e89d..0dbe8f5bec 100644 --- a/doc/src/Commands_fix.rst +++ b/doc/src/Commands_fix.rst @@ -39,6 +39,9 @@ OPT. * :doc:`ave/time ` * :doc:`aveforce ` * :doc:`balance ` + * :doc:`brownian ` + * :doc:`brownian/asphere ` + * :doc:`brownian/sphere ` * :doc:`bocs ` * :doc:`bond/break ` * :doc:`bond/create ` @@ -98,6 +101,7 @@ OPT. * :doc:`lb/viscous ` * :doc:`lineforce ` * :doc:`manifoldforce ` + * :doc:`mdi/engine ` * :doc:`meso/move ` * :doc:`momentum (k) ` * :doc:`momentum/chunk ` diff --git a/doc/src/Commands_pair.rst b/doc/src/Commands_pair.rst index accb79e82d..b3e29ec3d5 100644 --- a/doc/src/Commands_pair.rst +++ b/doc/src/Commands_pair.rst @@ -87,6 +87,8 @@ OPT. * :doc:`coul/wolf/cs ` * :doc:`dpd (gio) ` * :doc:`dpd/fdt ` + * :doc:`dpd/ext ` + * :doc:`dpd/ext/tstat ` * :doc:`dpd/fdt/energy (k) ` * :doc:`dpd/tstat (go) ` * :doc:`dsmc ` @@ -169,7 +171,7 @@ OPT. * :doc:`lj/sdk/coul/long (go) ` * :doc:`lj/sdk/coul/msm (o) ` * :doc:`lj/sf/dipole/sf (go) ` - * :doc:`lj/smooth (o) ` + * :doc:`lj/smooth (go) ` * :doc:`lj/smooth/linear (o) ` * :doc:`lj/switch3/coulgauss/long ` * :doc:`lj96/cut (go) ` diff --git a/doc/src/Developer_utils.rst b/doc/src/Developer_utils.rst index 17b4715dc7..44210218c0 100644 --- a/doc/src/Developer_utils.rst +++ b/doc/src/Developer_utils.rst @@ -9,14 +9,29 @@ reading or writing to files with error checking or translation of strings into specific types of numbers with checking for validity. This reduces redundant implementations and encourages consistent behavior. -I/O with status check -^^^^^^^^^^^^^^^^^^^^^ +I/O with status check and similar functions +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -These are wrappers around the corresponding C library calls like -``fgets()`` or ``fread()``. They will check if there were errors -on reading or an unexpected end-of-file state was reached. In that -case, the functions will stop the calculation with an error message, -indicating the name of the problematic file, if possible. +The the first two functions are wrappers around the corresponding C +library calls ``fgets()`` or ``fread()``. They will check if there +were errors on reading or an unexpected end-of-file state was reached. +In that case, the functions will stop with an error message, indicating +the name of the problematic file, if possible unless the *error* argument +is a NULL pointer. + +The :cpp:func:`fgets_trunc` function will work similar for ``fgets()`` +but it will read in a whole line (i.e. until the end of line or end +of file), but store only as many characters as will fit into the buffer +including a final newline character and the terminating NULL byte. +If the line in the file is longer it will thus be truncated in the buffer. +This function is used by :cpp:func:`read_lines_from_file` to read individual +lines but make certain they follow the size constraints. + +The :cpp:func:`read_lines_from_file` function will read the requested +number of lines of a maximum length into a buffer and will return 0 +if successful or 1 if not. It also guarantees that all lines are +terminated with a newline character and the entire buffer with a +NULL character. ---------- @@ -26,6 +41,12 @@ indicating the name of the problematic file, if possible. .. doxygenfunction:: sfread :project: progguide +.. doxygenfunction:: fgets_trunc + :project: progguide + +.. doxygenfunction:: read_lines_from_file + :project: progguide + ---------- String to number conversions with validity check @@ -164,7 +185,10 @@ Argument processing Convenience functions ^^^^^^^^^^^^^^^^^^^^^ -.. doxygenfunction:: logmesg +.. doxygenfunction:: logmesg(LAMMPS *lmp, const S &format, Args&&... args) + :project: progguide + +.. doxygenfunction:: logmesg(LAMMPS *lmp, const std::string &mesg) :project: progguide .. doxygenfunction:: getsyserror diff --git a/doc/src/Examples.rst b/doc/src/Examples.rst index 8a76dca66e..8bfc4ad573 100644 --- a/doc/src/Examples.rst +++ b/doc/src/Examples.rst @@ -108,6 +108,8 @@ Lowercase directories +-------------+------------------------------------------------------------------+ | msst | MSST shock dynamics | +-------------+------------------------------------------------------------------+ +| multi | multi neighboring for systems with large interaction disparities | ++-------------+------------------------------------------------------------------+ | nb3b | use of non-bonded 3-body harmonic pair style | +-------------+------------------------------------------------------------------+ | neb | nudged elastic band (NEB) calculation for barrier finding | diff --git a/doc/src/Howto.rst b/doc/src/Howto.rst index ff75b751b1..99a9fae4da 100644 --- a/doc/src/Howto.rst +++ b/doc/src/Howto.rst @@ -23,6 +23,7 @@ General howto Howto_library Howto_couple Howto_client_server + Howto_mdi Settings howto ============== diff --git a/doc/src/Howto_drude2.rst b/doc/src/Howto_drude2.rst index d95f5c02aa..24c21f27b1 100644 --- a/doc/src/Howto_drude2.rst +++ b/doc/src/Howto_drude2.rst @@ -9,6 +9,8 @@ USER-DRUDE package activated. Then, the data file and input scripts have to be modified to include the Drude dipoles and how to handle them. +Example input scripts available: examples/USER/drude + ---------- **Overview of Drude induced dipoles** diff --git a/doc/src/Howto_mdi.rst b/doc/src/Howto_mdi.rst new file mode 100644 index 0000000000..7cfcd01a50 --- /dev/null +++ b/doc/src/Howto_mdi.rst @@ -0,0 +1,132 @@ +Using LAMMPS with the MDI library for code coupling +=================================================== + +.. note:: + + This Howto doc page will eventually replace the + :doc:`Howto client/server ` doc page. + +Client/server coupling of two codes is where one code is the "client" +and sends request messages (data) to a "server" code. The server +responds to each request with a reply message. This enables the two +codes to work in tandem to perform a simulation. LAMMPS can act as +either a client or server code; it does this by using the `MolSSI +Driver Interface (MDI) library +`_, +developed by the `Molecular Sciences Software Institute (MolSSI) +`_. + +Alternate methods for code coupling with LAMMPS are described on the +:doc:`Howto couple ` doc page. + +Some advantages of client/server coupling are that the two codes can run +as stand-alone executables; they need not be linked together. Thus +neither code needs to have a library interface. This also makes it easy +to run the two codes on different numbers of processors. If a message +protocol (format and content) is defined for a particular kind of +simulation, then in principle any code which implements the client-side +protocol can be used in tandem with any code which implements the +server-side protocol. Neither code needs to know what specific other +code it is working with. + +In MDI nomenclature, a client code is the "driver", and a server code is +an "engine". One driver code can communicate with one or more instances +of one or more engine codes. Driver and engine codes can be written in +any language: C, C++, Fortran, Python, etc. + +In addition to allowing driver and engine(s) running to run as +stand-alone executables, MDI also enables a server code to be a +"plugin" to the client code. In this scenario, server code(s) are +compiled as shared libraries, and one (or more) instances of the +server are instantiated by the driver code. If the driver code runs +in parallel, it can split its MPI communicator into multiple +sub-communicators, and launch each plugin engine instance on a +sub-communicator. Driver processors in that sub-communicator exchange +messages with that engine instance, and can also send MPI messages to +other processors in the driver. The driver code can also destroy +engine instances and re-instantiate them. + +The way that a driver communicates with an engine is by making +MDI_Send() and MDI_Recv() calls, which are conceptually similar to +MPI_Send() and MPI_Recv() calls. Each send or receive has a string +which identifies the command name, and optionally some data, which can +be a single value or vector of values of any data type. Inside the +MDI library, data is exchanged between the driver and engine via MPI +calls or sockets. This a run-time choice by the user. + +------------- + +As an example, LAMMPS and the ``pw.x`` command from Quantum Espresso (a +suite of quantum DFT codes), can work together via the MDI library to +perform an ab initio MD (AIMD) simulation, where LAMMPS runs an MD +simulation and sends a message each timestep to ``pw.x`` asking it to +compute quantum forces on the current configuration of atoms. Here is +how the 2 codes are launched to communicate by MPI: + +.. code-block:: bash + + % mpirun -np 2 lmp_mpi -mdi "-role DRIVER -name d -method MPI" \ + -in in.aimd : -np 16 pw.x -in qe.in -mdi "-role ENGINE -name e -method MPI" + +In this case LAMMPS runs on 2 processors (MPI tasks), ``pw.x`` runs on 16 +processors. + +Here is how the 2 codes are launched to communicate by sockets: + +.. code-block:: bash + + % mpirun -np 2 lmp_mpi -mdi "-role DRIVER -name d -method TCP -port 8021" -in in.aimd + % mpirun -np 16 pw.x -in qe.in -mdi "-role ENGINE -name e -method TCP -port 8021 -hostname localhost" + +These commands could be issued in different windows on a desktop +machine. Or in the same window, if the first command is ended with +"&" so as to run in the background. If "localhost" is replaced by an +IP address, ``pw.x`` could be run on another machine on the same network, or +even on another machine across the country. + +After both codes initialize themselves to model the same system, this is +what occurs each timestep: + +* LAMMPS send a ">COORDS" message to ``pw.x`` with a 3*N vector of current atom coords +* ``pw.x`` receives the message/coords and computes quantum forces on all the atoms +* LAMMPS send a "` command. This will put LAMMPS into +"engine mode" where it waits for messages and data from the driver. +When the driver sends an "EXIT" command, LAMMPS will exit engine mode +and the input script will continue. + +If LAMMPS is used as a plugin engine it operates the same way, except +that the driver will pass LAMMPS an input script to initialize itself. +Upon receiving the "EXIT" command, LAMMPS will exit engine mode and the +input script will continue. After finishing execution of the input +script, the instance of LAMMPS will be destroyed. + +LAMMPS supports the full set of MD-appropriate engine commands defined +by the MDI library. See the :doc:`mdi/engine ` doc page for +a list of these. + +If those commands are not sufficient for a user-developed driver to use +LAMMPS as an engine, then new commands can be easily added. See these +two files which implement the definition of MDI commands and the logic +for responding to them: + +* src/MDI/mdi_engine.cpp +* src/MDI/fix_mdi_engine.cpp diff --git a/doc/src/Library_config.rst b/doc/src/Library_config.rst index a5b87e23a7..04b12f35de 100644 --- a/doc/src/Library_config.rst +++ b/doc/src/Library_config.rst @@ -15,6 +15,8 @@ This section documents the following functions: - :cpp:func:`lammps_config_package_count` - :cpp:func:`lammps_config_package_name` - :cpp:func:`lammps_config_accelerator` +- :cpp:func:`lammps_has_gpu_device` +- :cpp:func:`lammps_gpu_device_info` - :cpp:func:`lammps_has_style` - :cpp:func:`lammps_style_count` - :cpp:func:`lammps_style_name` @@ -132,6 +134,16 @@ approach. ----------------------- +.. doxygenfunction:: lammps_has_gpu_device + :project: progguide + +----------------------- + +.. doxygenfunction:: lammps_get_gpu_device_info + :project: progguide + +----------------------- + .. doxygenfunction:: lammps_has_style :project: progguide diff --git a/doc/src/Manual.rst b/doc/src/Manual.rst index 9198f545bc..4c7e5fa8f5 100644 --- a/doc/src/Manual.rst +++ b/doc/src/Manual.rst @@ -89,10 +89,15 @@ Programmer Guide Modify Developer +***************** +Command Reference +***************** + +.. _command_reference: .. toctree:: - :caption: Index - :name: index - :hidden: + :name: reference + :maxdepth: 1 + :caption: Command Reference commands_list fixes @@ -118,7 +123,7 @@ Indices and tables .. admonition:: Web Browser Compatibility The HTML version of the manual makes use of advanced features present - int "modern" web browsers. This can lead to incompatibilities with older + in "modern" web browsers. This can lead to incompatibilities with older web browsers (released more than 4 years ago) and specific vendor browsers (e.g. Internet Explorer on Windows; Microsoft Edge works well though) where parts of the pages are not rendered as expected (e.g. the layout is diff --git a/doc/src/Modify_contribute.rst b/doc/src/Modify_contribute.rst index 4998712a3d..875212be4e 100644 --- a/doc/src/Modify_contribute.rst +++ b/doc/src/Modify_contribute.rst @@ -10,15 +10,15 @@ Tutorial ` for a demonstration on how to do that. An alternative is to contact the LAMMPS developers or the indicated developer of a package or feature directly and send in your contribution via e-mail, but that can add a significant delay on getting your -contribution included, depending on how busy the developer is you -contact, how complex a task it would be to integrate that code, and how +contribution included, depending on how busy the respective developer +is, how complex a task it would be to integrate that code, and how many - if any - changes are required before the code can be included. For any larger modifications or programming project, you are encouraged -to contact the LAMMPS developers ahead of time, in order to discuss -implementation strategies and coding guidelines, that will make it -easier to integrate your contribution and result in less work for -everybody involved. You are also encouraged to search through the list +to contact the LAMMPS developers ahead of time in order to discuss +implementation strategies and coding guidelines. That will make it +easier to integrate your contribution and results in less work for +everybody involved. You are also encouraged to search through the list of `open issues on GitHub `_ and submit a new issue for a planned feature, so you would not duplicate the work of others (and possibly get scooped by them) or have your work @@ -34,18 +34,19 @@ installing, or using LAMMPS. Please contact the `lammps-users mailing list `_ for those purposes instead. How quickly your contribution will be integrated depends largely on how -much effort it will cause to integrate and test it, how much it requires -changes to the core codebase, and of how much interest it is to the -larger LAMMPS community. Please see below for a checklist of typical -requirements. Once you have prepared everything, see the :doc:`LAMMPS GitHub -Tutorial ` page for instructions on -how to submit your changes or new files through a GitHub pull -request. If you prefer to submit patches or full files, you should first -make certain, that your code works correctly with the latest patch-level -version of LAMMPS and contains all bug fixes from it. Then create a -gzipped tar file of all changed or added files or a corresponding patch -file using 'diff -u' or 'diff -c' and compress it with gzip. Please only -use gzip compression, as this works well on all platforms. +much effort it will cause to integrate and test it, how many and what +kind of changes it requires to the core codebase, and of how much +interest it is to the larger LAMMPS community. Please see below for a +checklist of typical requirements. Once you have prepared everything, +see the :doc:`LAMMPS GitHub Tutorial ` page for +instructions on how to submit your changes or new files through a GitHub +pull request. If you prefer to submit patches or full files, you should +first make certain, that your code works correctly with the latest +patch-level version of LAMMPS and contains all bug fixes from it. Then +create a gzipped tar file of all changed or added files or a +corresponding patch file using 'diff -u' or 'diff -c' and compress it +with gzip. Please only use gzip compression, as this works well and is +available on all platforms. If the new features/files are broadly useful we may add them as core files to LAMMPS or as part of a :doc:`standard package `. Else we will add them as a @@ -57,9 +58,12 @@ added to the LAMMPS distribution. All the standard and user packages are listed and described on the :doc:`Packages details ` doc page. Note that by providing us files to release, you are agreeing to make -them open-source, i.e. we can release them under the terms of the GPL, -used as a license for the rest of LAMMPS. See the :doc:`LAMMPS license -` doc page for details. +them open-source, i.e. we can release them under the terms of the GPL +(version 2), used as a license for the rest of LAMMPS. And as part of +a LGPL (version 2.1) distribution that we make available to developers +on request only and with files that are authorized for that kind of +distribution removed (e.g. interface to FFTW). See the +:doc:`LAMMPS license ` doc page for details. With user packages and files, all we are really providing (aside from the fame and fortune that accompanies having your name in the source @@ -112,7 +116,7 @@ packages in the src directory for examples. If you are uncertain, please ask. your contribution(s) to be added to main LAMMPS code or one of its standard packages, it needs to be written in a style compatible with other LAMMPS source files. This means: 2-character indentation per - level, **no tabs**\ , no lines over 80 characters. I/O is done via + level, **no tabs**\ , no lines over 100 characters. I/O is done via the C-style stdio library (mixing of stdio and iostreams is generally discouraged), class header files should not import any system headers outside of , STL containers should be avoided in headers, @@ -131,6 +135,31 @@ packages in the src directory for examples. If you are uncertain, please ask. LAMMPS source files, including the use of the error class for error and warning messages. +* To simplify reformatting contributed code in a way that is compatible + with the LAMMPS formatting styles, you can use clang-format (version 8 + or later). The LAMMPS distribution includes a suitable ``.clang-format`` + file which will be applied if you run ``clang-format -i some_file.cpp`` + on your files inside the LAMMPS src tree. Please only reformat files + that you have contributed. For header files containing a + ``SomeStyle(keyword, ClassName)`` macros it is required to have this + macro embedded with a pair of ``// clang-format off``, ``// clang-format on`` + commends and the line must be terminated with a semi-colon (;). + Example: + + .. code-block:: c++ + + #ifdef COMMAND_CLASS + // clang-format off + CommandStyle(run,Run); + // clang-format on + #else + + #ifndef LMP_RUN_H + [...] + + You may also use ``// clang-format on/off`` throughout your file + to protect sections of the file from being reformatted. + * If you want your contribution to be added as a user-contributed feature, and it's a single file (actually a \*.cpp and \*.h file) it can rapidly be added to the USER-MISC directory. Send us the one-line diff --git a/doc/src/Packages_details.rst b/doc/src/Packages_details.rst index f1c590d850..6ee9a56513 100644 --- a/doc/src/Packages_details.rst +++ b/doc/src/Packages_details.rst @@ -69,6 +69,7 @@ page gives those details. * :ref:`USER-ATC ` * :ref:`USER-AWPMD ` * :ref:`USER-BOCS ` + * :ref:`USER-BROWNIAN ` * :ref:`USER-CGDNA ` * :ref:`USER-CGSDK ` * :ref:`USER-COLVARS ` @@ -81,6 +82,7 @@ page gives those details. * :ref:`USER-INTEL ` * :ref:`USER-LB ` * :ref:`USER-MANIFOLD ` + * :ref:`USER-MDI ` * :ref:`USER-MEAMC ` * :ref:`USER-MESODPD ` * :ref:`USER-MESONT ` @@ -445,7 +447,7 @@ time via the "-sf kk" or "-suffix kk" :doc:`command-line switches ` :ref:`USER-INTEL `, and :ref:`USER-OMP ` packages, which have styles optimized for CPUs, KNLs, and GPUs. -You must have a C++11 compatible compiler to use this package. +You must have a C++14 compatible compiler to use this package. KOKKOS makes extensive use of advanced C++ features, which can expose compiler bugs, especially when compiling for maximum performance at high optimization levels. Please see the file @@ -1266,6 +1268,26 @@ Example inputs are in the examples/USER/bocs folder. ---------- +.. _PKG-USER-BROWNIAN: + +USER-BROWNIAN package +--------------------- + +**Contents:** + +This package provides :doc:`fix brownian, fix brownian/sphere, and +fix brownian/asphere ` as well as +:doc:`fix propel/self ` which allow to do Brownian +Dynamics time integration of point, spherical and aspherical particles +and also support self-propelled particles. + +**Authors:** Sam Cameron (University of Bristol), +Stefan Paquay (while at Brandeis University) (initial version of fix propel/self) + +Example inputs are in the examples/USER/brownian folder. + +---------- + .. _PKG-USER-CGDNA: USER-CGDNA package @@ -1770,6 +1792,28 @@ Waltham, MA, USA) ---------- +.. _PKG-USER-MDI: + +USER-MDI package +---------------- + +**Contents:** + +A LAMMPS command and fix to allow client-server coupling of LAMMPS to +other atomic or molecular simulation codes via the `MolSSI Driver Interface +(MDI) library `_. + +**Author:** Taylor Barnes - MolSSI, taylor.a.barnes at gmail.com + +**Supporting info:** + +* src/USER-MDI/README +* :doc:`mdi/engine ` +* :doc:`fix mdi/engine ` +* examples/USER/mdi + +---------- + .. _PKG-USER-MEAMC: USER-MEAMC package diff --git a/doc/src/Packages_user.rst b/doc/src/Packages_user.rst index 00d1dfb67b..f14d449dfd 100644 --- a/doc/src/Packages_user.rst +++ b/doc/src/Packages_user.rst @@ -39,6 +39,8 @@ package: +------------------------------------------------+-----------------------------------------------------------------+-------------------------------------------------------------------------------+------------------------------------------------------+---------+ | :ref:`USER-BOCS ` | BOCS bottom up coarse graining | :doc:`fix bocs ` | USER/bocs | no | +------------------------------------------------+-----------------------------------------------------------------+-------------------------------------------------------------------------------+------------------------------------------------------+---------+ +| :ref:`USER-BROWNIAN ` | Brownian dynamics and self-propelled particles | :doc:`fix brownian `, :doc:`fix propel/self ` | USER/brownian | no | ++------------------------------------------------+-----------------------------------------------------------------+-------------------------------------------------------------------------------+------------------------------------------------------+---------+ | :ref:`USER-CGDNA ` | coarse-grained DNA force fields | src/USER-CGDNA/README | USER/cgdna | no | +------------------------------------------------+-----------------------------------------------------------------+-------------------------------------------------------------------------------+------------------------------------------------------+---------+ | :ref:`USER-CGSDK ` | SDK coarse-graining model | :doc:`pair_style lj/sdk ` | USER/cgsdk | no | @@ -63,6 +65,8 @@ package: +------------------------------------------------+-----------------------------------------------------------------+-------------------------------------------------------------------------------+------------------------------------------------------+---------+ | :ref:`USER-MANIFOLD ` | motion on 2d surfaces | :doc:`fix manifoldforce ` | USER/manifold | no | +------------------------------------------------+-----------------------------------------------------------------+-------------------------------------------------------------------------------+------------------------------------------------------+---------+ +| :ref:`USER-MDI ` | client-server coupling | :doc:`MDI Howto ` | USER/mdi | ext | ++------------------------------------------------+-----------------------------------------------------------------+-------------------------------------------------------------------------------+------------------------------------------------------+---------+ | :ref:`USER-MEAMC ` | modified EAM potential (C++) | :doc:`pair_style meam/c ` | meamc | no | +------------------------------------------------+-----------------------------------------------------------------+-------------------------------------------------------------------------------+------------------------------------------------------+---------+ | :ref:`USER-MESODPD ` | mesoscale DPD models | :doc:`pair_style edpd ` | USER/mesodpd | no | diff --git a/doc/src/Run_options.rst b/doc/src/Run_options.rst index 25756d9be0..d8adfcc5be 100644 --- a/doc/src/Run_options.rst +++ b/doc/src/Run_options.rst @@ -10,6 +10,7 @@ letter abbreviation can be used: * :ref:`-i or -in ` * :ref:`-k or -kokkos ` * :ref:`-l or -log ` +* :ref:`-mdi ` * :ref:`-m or -mpicolor ` * :ref:`-c or -cite ` * :ref:`-nc or -nocite ` @@ -196,9 +197,23 @@ Option -plog will override the name of the partition log files file.N. ---------- +.. _mdi: + +**-mdi 'multiple flags'** + +This flag is only recognized and used when LAMMPS has support for the MolSSI +Driver Interface (MDI) included as part of the :ref:`USER-MDI ` +package. This flag is specific to the MDI library and controls how LAMMPS +interacts with MDI. There are usually multiple flags that have to follow it +and those have to be placed in quotation marks. For more information about +how to launch LAMMPS in MDI client/server mode please refer to the +:doc:`MDI Howto `. + +---------- + .. _mpicolor: -**-mpicolor** color +**-mpicolor color** If used, this must be the first command-line argument after the LAMMPS executable name. It is only used when LAMMPS is launched by an mpirun @@ -223,7 +238,7 @@ links with from the lib/message directory. See the .. _cite: -**-cite style or file name** +**-cite style** or **file name** Select how and where to output a reminder about citing contributions to the LAMMPS code that were used during the run. Available styles are diff --git a/doc/src/Tools.rst b/doc/src/Tools.rst index 376489dcce..a790c33ec2 100644 --- a/doc/src/Tools.rst +++ b/doc/src/Tools.rst @@ -94,6 +94,7 @@ Miscellaneous tools * :ref:`kate ` * :ref:`LAMMPS shell ` * :ref:`LAMMPS magic patterns for file(1) ` + * :ref:`Offline build tool ` * :ref:`singularity ` * :ref:`SWIG interface ` * :ref:`vim ` @@ -756,6 +757,103 @@ See the README file in the tools/msi2lmp folder for more information. ---------- +.. _offline: + +Scripts for building LAMMPS when offline +---------------------------------------- + +In some situations it might be necessary to build LAMMPS on a system +without direct internet access. The scripts in ``tools/offline`` folder +allow you to pre-load external dependencies for both the documentation +build and for building LAMMPS with CMake. + +It does so by + + #. downloading necessary ``pip`` packages, + #. cloning ``git`` repositories + #. downloading tarballs + +to a designated cache folder. + +As of April 2021, all of these downloads make up around 600MB. By +default, the offline scripts will download everything into the +``$HOME/.cache/lammps`` folder, but this can be changed by setting the +``LAMMPS_CACHING_DIR`` environment variable. + +Once the caches have been initialized, they can be used for building the +LAMMPS documentation or compiling LAMMPS using CMake on an offline +system. + +The ``use_caches.sh`` script must be sourced into the current shell +to initialize the offline build environment. Note that it must use +the same ``LAMMPS_CACHING_DIR``. This script does the following: + + #. Set up environment variables that modify the behavior of both, + ``pip`` and ``git`` + #. Start a simple local HTTP server using Python to host files for CMake + +Afterwards, it will print out instruction on how to modify the CMake +command line to make sure it uses the local HTTP server. + +To undo the environment changes and shutdown the local HTTP server, +run the ``deactivate_caches`` command. + +Examples +^^^^^^^^ + +For all of the examples below, you first need to create the cache, which +requires an internet connection. + +.. code-block:: bash + + ./tools/offline/init_caches.sh + +Afterwards, you can disconnect or copy the contents of the +``LAMMPS_CACHING_DIR`` folder to an offline system. + +Documentation Build +^^^^^^^^^^^^^^^^^^^ + +The documentation build will create a new virtual environment that +typically first installs dependencies from ``pip``. With the offline +environment loaded, these installations will instead grab the necessary +packages from your local cache. + +.. code-block:: bash + + # if LAMMPS_CACHING_DIR is different from default, make sure to set it first + # export LAMMPS_CACHING_DIR=path/to/folder + source tools/offline/use_caches.sh + cd doc/ + make html + + deactivate_caches + +CMake Build +^^^^^^^^^^^ + +When compiling certain packages with external dependencies, the CMake +build system will download necessary files or sources from the web. For +more flexibility the CMake configuration allows users to specify the URL +of each of these dependencies. What the ``init_caches.sh`` script does +is create a CMake "preset" file, which sets the URLs for all of the known +dependencies and redirects the download to the local cache. + +.. code-block:: bash + + # if LAMMPS_CACHING_DIR is different from default, make sure to set it first + # export LAMMPS_CACHING_DIR=path/to/folder + source tools/offline/use_caches.sh + + mkdir build + cd build + cmake -D LAMMPS_DOWNLOADS_URL=${HTTP_CACHE_URL} -C "${LAMMPS_HTTP_CACHE_CONFIG}" -C ../cmake/presets/most.cmake ../cmake + make -j 8 + + deactivate_caches + +---------- + .. _phonon: phonon tool diff --git a/doc/src/angle_table.rst b/doc/src/angle_table.rst index cad9f5a82f..b8aeae780b 100644 --- a/doc/src/angle_table.rst +++ b/doc/src/angle_table.rst @@ -69,7 +69,6 @@ parenthesized comments): HAM (keyword is the first text on line) N 181 FP 0 0 EQ 90.0 (N, FP, EQ parameters) (blank line) - N 181 FP 0 0 (N, FP parameters) 1 0.0 200.5 2.5 (index, angle, energy, derivative) 2 1.0 198.0 2.5 ... diff --git a/doc/src/comm_modify.rst b/doc/src/comm_modify.rst index 864c544fed..27b370c941 100644 --- a/doc/src/comm_modify.rst +++ b/doc/src/comm_modify.rst @@ -11,13 +11,17 @@ Syntax comm_modify keyword value ... * zero or more keyword/value pairs may be appended -* keyword = *mode* or *cutoff* or *cutoff/multi* or *group* or *vel* +* keyword = *mode* or *cutoff* or *cutoff/multi* or *multi/reduce* or *group* or *vel* .. parsed-literal:: - *mode* value = *single* or *multi* = communicate atoms within a single or multiple distances + *mode* value = *single*, *multi*, or *multi/old* = communicate atoms within a single or multiple distances *cutoff* value = Rcut (distance units) = communicate atoms from this far away - *cutoff/multi* type value + *cutoff/multi* collection value + collection = atom collection or collection range (supports asterisk notation) + value = Rcut (distance units) = communicate atoms for selected types from this far away + *reduce/multi* arg = none = reduce number of communicated ghost atoms for multi style + *cutoff/multi/old* type value type = atom type or type range (supports asterisk notation) value = Rcut (distance units) = communicate atoms for selected types from this far away *group* value = group-ID = only communicate atoms in the group @@ -28,9 +32,9 @@ Examples .. code-block:: LAMMPS - comm_modify mode multi + comm_modify mode multi reduce/multi comm_modify mode multi group solvent - comm_modift mode multi cutoff/multi 1 10.0 cutoff/multi 2*4 15.0 + comm_modify mode multi cutoff/multi 1 10.0 cutoff/multi 2*4 15.0 comm_modify vel yes comm_modify mode single cutoff 5.0 vel yes comm_modify cutoff/multi * 0.0 @@ -62,12 +66,18 @@ sub-domain. The distance is by default the maximum of the neighbor cutoff across all atom type pairs. For many systems this is an efficient algorithm, but for systems with -widely varying cutoffs for different type pairs, the *multi* mode can -be faster. In this case, each atom type is assigned its own distance +widely varying cutoffs for different type pairs, the *multi* or *multi/old* mode can +be faster. In *multi*, each atom is assigned to a collection which should +correspond to a set of atoms with similar interaction cutoffs. +In this case, each atom collection is assigned its own distance cutoff for communication purposes, and fewer atoms will be -communicated. See the :doc:`neighbor multi ` command for a -neighbor list construction option that may also be beneficial for -simulations of this kind. +communicated. in *multi/old*, a similar technique is used but atoms +are grouped by atom type. See the :doc:`neighbor multi ` and +:doc:`neighbor multi/old ` commands for +neighbor list construction options that may also be beneficial for +simulations of this kind. The *multi* communication mode is only compatible +with the *multi* neighbor style. The *multi/old* communication mode is comparable +with both the *multi* and *multi/old* neighbor styles. The *cutoff* keyword allows you to extend the ghost cutoff distance for communication mode *single*\ , which is the distance from the borders @@ -87,12 +97,26 @@ warning is printed, if this bond based estimate is larger than the communication cutoff used. The *cutoff/multi* option is equivalent to *cutoff*\ , but applies to -communication mode *multi* instead. Since in this case the communication -cutoffs are determined per atom type, a type specifier is needed and -cutoff for one or multiple types can be extended. Also ranges of types -using the usual asterisk notation can be given. For granular pair styles, -the default cutoff is set to the sum of the current maximum atomic radii -for each type. +communication mode *multi* instead. Since the communication cutoffs are +determined per atom collections, a collection specifier is needed and +cutoff for one or multiple collections can be extended. Also ranges of +collections using the usual asterisk notation can be given. Collections +are indexed from 1 to N where N is the total number of collections. +Note that the arguments for *cutoff/multi* are parsed right before each +simulation to account for potential changes in the number of +collections. Custom cutoffs are preserved between runs but if +collections are redefined, one may want to re-specify the communication +cutoffs. For granular pair styles,the default cutoff is set to the sum +of the current maximum atomic radii for each collection. The +*cutoff/multi/old* option is similar to *cutoff/multi* except it +operates on atom types as opposed to collections. + +The *reduce/multi* option applies to *multi* and sets the communication +cutoff for a particle equal to the maximum interaction distance between particles +in the same collection. This reduces the number of +ghost atoms that need to be communicated. This method is only compatible with the +*multi* neighbor style and requires a half neighbor list and Newton on. +See the :doc:`neighbor multi ` command for more information. These are simulation scenarios in which it may be useful or even necessary to set a ghost cutoff > neighbor cutoff: @@ -123,7 +147,7 @@ ghost cutoff should be set. In the last scenario, a :doc:`fix ` or :doc:`compute ` or :doc:`pairwise potential ` needs to calculate with ghost atoms beyond the normal pairwise cutoff for some computation it -performs (e.g. locate neighbors of ghost atoms in a multibody pair +performs (e.g. locate neighbors of ghost atoms in a manybody pair potential). Setting the ghost cutoff appropriately can insure it will find the needed atoms. diff --git a/doc/src/commands_list.rst b/doc/src/commands_list.rst index e30d5c52dc..75affe7ce6 100644 --- a/doc/src/commands_list.rst +++ b/doc/src/commands_list.rst @@ -59,6 +59,7 @@ Commands lattice log mass + mdi_engine message min_modify min_spin diff --git a/doc/src/compute_fep.rst b/doc/src/compute_fep.rst index 279148a55d..ef908acfc0 100644 --- a/doc/src/compute_fep.rst +++ b/doc/src/compute_fep.rst @@ -48,6 +48,8 @@ Examples compute 1 all fep 298 pair lj/cut epsilon 1 * v_delta pair lj/cut sigma 1 * v_delta volume yes compute 1 all fep 300 atom charge 2 v_delta +Example input scripts available: examples/USER/fep + Description """"""""""" diff --git a/doc/src/compute_temp_drude.rst b/doc/src/compute_temp_drude.rst index 418f2e1e5d..7e0f5f6555 100644 --- a/doc/src/compute_temp_drude.rst +++ b/doc/src/compute_temp_drude.rst @@ -20,6 +20,8 @@ Examples compute TDRUDE all temp/drude +Example input scripts available: examples/USER/drude + Description """"""""""" diff --git a/doc/src/fix.rst b/doc/src/fix.rst index 109bfb00be..d54594d5af 100644 --- a/doc/src/fix.rst +++ b/doc/src/fix.rst @@ -182,6 +182,9 @@ accelerated styles exist. * :doc:`ave/time ` - compute/output global time-averaged quantities * :doc:`aveforce ` - add an averaged force to each atom * :doc:`balance ` - perform dynamic load-balancing +* :doc:`brownian ` - overdamped translational brownian motion +* :doc:`brownian/asphere ` - overdamped translational and rotational brownian motion for ellipsoids +* :doc:`brownian/sphere ` - overdamped translational and rotational brownian motion for spheres * :doc:`bocs ` - NPT style time integration with pressure correction * :doc:`bond/break ` - break bonds on the fly * :doc:`bond/create ` - create bonds on the fly @@ -241,6 +244,7 @@ accelerated styles exist. * :doc:`lb/viscous ` - * :doc:`lineforce ` - constrain atoms to move in a line * :doc:`manifoldforce ` - restrain atoms to a manifold during minimization +* :doc:`mdi/engine ` - connect LAMMPS to external programs via the MolSSI Driver Interface (MDI) * :doc:`meso/move ` - move mesoscopic SPH/SDPD particles in a prescribed fashion * :doc:`momentum ` - zero the linear and/or angular momentum of a group of atoms * :doc:`momentum/chunk ` - zero the linear and/or angular momentum of a chunk of atoms diff --git a/doc/src/fix_adapt_fep.rst b/doc/src/fix_adapt_fep.rst index b45ac80429..81a34ab75e 100644 --- a/doc/src/fix_adapt_fep.rst +++ b/doc/src/fix_adapt_fep.rst @@ -56,6 +56,9 @@ Examples fix 1 all adapt/fep 1 pair lj/cut epsilon * * v_scale1 coul/cut scale 3 3 v_scale2 scale yes reset yes fix 1 all adapt/fep 10 atom diameter 1 v_size + +Example input scripts available: examples/USER/fep + Description """"""""""" diff --git a/doc/src/fix_bond_react.rst b/doc/src/fix_bond_react.rst index 5291aff81b..f42952ecdc 100644 --- a/doc/src/fix_bond_react.rst +++ b/doc/src/fix_bond_react.rst @@ -328,8 +328,8 @@ keyword 'ChiralIDs' lists the atom IDs of chiral atoms whose handedness should be enforced. The fifth optional section begins with the keyword 'Constraints' and lists additional criteria that must be satisfied in order for the reaction to occur. Currently, there are -five types of constraints available, as discussed below: 'distance', -'angle', 'dihedral', 'arrhenius', and 'rmsd'. +six types of constraints available, as discussed below: 'distance', +'angle', 'dihedral', 'arrhenius', 'rmsd', and 'custom'. A sample map file is given below: @@ -500,6 +500,45 @@ example, the molecule fragment could consist of only the backbone atoms of a polymer chain. This constraint can be used to enforce a specific relative position and orientation between reacting molecules. +The constraint of type 'custom' has the following syntax: + +.. parsed-literal:: + + custom *varstring* + +where 'custom' is the required keyword, and *varstring* is a +variable expression. The expression must be a valid equal-style +variable formula that can be read by the :doc:`variable ` command, +after any special reaction functions are evaluated. If the resulting +expression is zero, the reaction is prevented from occurring; +otherwise, it is permitted to occur. There are two special reaction +functions available, 'rxnsum' and 'rxnave'. These functions operate +over the atoms in a given reaction site, and have one mandatory +argument and one optional argument. The mandatory argument is the +identifier for an atom-style variable. The second, optional argument +is the name of a molecule fragment in the pre-reaction template, and +can be used to operate over a subset of atoms in the reaction site. +The 'rxnsum' function sums the atom-style variable over the reaction +site, while the 'rxnave' returns the average value. For example, a +constraint on the total potential energy of atoms involved in the +reaction can be imposed as follows: + +.. code-block:: LAMMPS + +compute 1 all pe/atom # in LAMMPS input script +variable my_pe atom c_1 # in LAMMPS input script + +.. code-block:: LAMMPS + +custom "rxnsum(v_my_pe) > 100" # in Constraints section of map file + +The above example prevents the reaction from occurring unless the +total potential energy of the reaction site is above 100. The variable +expression can be interpreted as the probability of the reaction +occurring by using an inequality and the 'random(x,y,z)' function +available as an equal-style variable input, similar to the 'arrhenius' +constraint above. + By default, all constraints must be satisfied for the reaction to occur. In other words, constraints are evaluated as a series of logical values using the logical AND operator "&&". More complex logic diff --git a/doc/src/fix_brownian.rst b/doc/src/fix_brownian.rst new file mode 100644 index 0000000000..d032346617 --- /dev/null +++ b/doc/src/fix_brownian.rst @@ -0,0 +1,216 @@ +.. index:: fix brownian +.. index:: fix brownian/sphere +.. index:: fix brownian/asphere + +fix brownian command +=========================== + +fix brownian/sphere command +=========================== + +fix brownian/sphere command +=========================== + + +Syntax +"""""" + +.. parsed-literal:: + + fix ID group-ID style_name temp seed keyword args + +* ID, group-ID are documented in :doc:`fix ` command +* style_name = *brownian* or *brownian/sphere* or *brownian/asphere* +* temp = temperature +* seed = random number generator seed +* one or more keyword/value pairs may be appended +* keyword = *rng* or *dipole* or *gamma_r_eigen* or *gamma_t_eigen* or *gamma_r* or *gamma_t* + + .. parsed-literal:: + + *rng* value = *uniform* or *gaussian* or *none* + *uniform* = use uniform random number generator + *gaussian* = use gaussian random number generator + *none* = turn off noise + *dipole* value = *mux* and *muy* and *muz* for *brownian/asphere* + *mux*, *muy*, and *muz* = update orientation of dipole having direction (*mux*,*muy*,*muz*) in body frame of rigid body + *gamma_r_eigen* values = *gr1* and *gr2* and *gr3* for *brownian/asphere* + *gr1*, *gr2*, and *gr3* = diagonal entries of body frame rotational friction tensor + *gamma_r* values = *gr* for *brownian/sphere* + *gr* = magnitude of the (isotropic) rotational friction tensor + *gamma_t_eigen* values = *gt1* and *gt2* and *gt3* for *brownian/asphere* + *gt1*, *gt2*, and *gt3* = diagonal entries of body frame translational friction tensor + *gamma_t* values = *gt* for *brownian* and *brownian/sphere* + *gt* = magnitude of the (isotropic) translational friction tensor + + +Examples +"""""""" + +.. code-block:: LAMMPS + + fix 1 all brownian 1.0 12908410 gamma_t 1.0 + fix 1 all brownian 1.0 12908410 gamma_t 3.0 rng gaussian + fix 1 all brownian/sphere 1.0 1294019 gamma_t 3.0 gamma_r 1.0 + fix 1 all brownian/sphere 1.0 19581092 gamma_t 1.0 gamma_r 0.3 rng none + fix 1 all brownian/asphere 1.0 1294019 gamma_t_eigen 1.0 2.0 3.0 gamma_r_eigen 4.0 7.0 8.0 rng gaussian + fix 1 all brownian/asphere 1.0 1294019 gamma_t_eigen 1.0 2.0 3.0 gamma_r_eigen 4.0 7.0 8.0 dipole 1.0 0.0 0.0 + + +Description +""""""""""" + +Perform Brownian Dynamics time integration to update position, velocity, +dipole orientation (for spheres) and quaternion orientation (for +ellipsoids, with optional dipole update as well) of all particles in the +fix group in each timestep. Brownian Dynamics uses Newton's laws of +motion in the limit that inertial forces are negligible compared to +viscous forces. The stochastic equation of motion for the center of mass +positions is + +.. math:: + + d\mathbf{r} = \mathbf{\gamma}_t^{-1}\mathbf{F}dt+\sqrt{2k_BT}\mathbf{\gamma}_t^{-1/2}d\mathbf{W}_t, + +in the lab-frame (i.e. :math:`\mathbf{\gamma}_t` is not diagonal, but +only depends on orientation and so the noise is still additive). + +The rotational motion for the spherical and ellipsoidal particles is not +as simple an expression, but is chosen to replicate the Boltzmann +distribution for the case of conservative torques (see :ref:`(Ilie) +` or :ref:`(Delong) `). + +For the style *brownian*, only the positions of the particles are +updated. This is therefore suitable for point particle simulations. + +For the style *brownian/sphere*, the positions of the particles are +updated, and a dipole slaved to the spherical orientation is also +updated. This style therefore requires the hybrid atom style +:doc:`atom_style dipole ` and :doc:`atom_style sphere +`. + +For the style *brownian/asphere*, the center of mass positions and the +quaternions of ellipsoidal particles are updated. This fix style is +suitable for equations of motion where the rotational and translational +friction tensors can be diagonalized in a certain (body) reference frame. + + +--------- + +.. note:: + + This integrator does not by default assume a relationship between the + rotational and translational friction tensors, though such a relationship + should exist in the case of no-slip boundary conditions between the particles and + the surrounding (implicit) solvent. E.g. in the case of spherical particles, + the condition :math:`\gamma_t=3\gamma_r/\sigma^2` must be explicitly + accounted for by setting *gamma_t* to 3x and *gamma_r* to x (where + :math:`\sigma` is the spherical diameter). A similar (though more complex) + relationship holds for ellipsoids and rod-like particles. + +--------- + +.. note:: + + Temperature computation using the :doc:`compute temp ` + will not correctly compute temperature of these overdamped dynamics + since we are explicitly neglecting inertial effects. + Furthermore, this time integrator does not add the stochastic terms or + viscous terms to the force and/or torques. Rather, they are just added + in to the equations of motion to update the degrees of freedom. + +--------- + + +If the *rng* keyword is used with the *uniform* value, then the noise +is generated from a uniform distribution (see +:ref:`(Dunweg) ` for why this works). This is the same method +of noise generation as used in :doc:`fix_langevin `. + +If the *rng* keyword is used with the *gaussian* value, then the noise +is generated from a gaussian distribution. Typically this added +complexity is unnecessary, and one should be fine using the *uniform* +value for reasons argued in :ref:`(Dunweg) `. + +If the *rng* keyword is used with the *none* value, then the noise +terms are set to zero. + +The *gamma_t* keyword sets the (isotropic) translational viscous damping. +Required for (and only compatible with) *brownian* and *brownian/sphere*. +The units of *gamma_t* are mass/time. + +The *gamma_r* keyword sets the (isotropic) rotational viscous damping. +Required for (and only compatible with) *brownian/sphere*. +The units of *gamma_r* are mass*length**2/time. + +The *gamma_r_eigen*, and *gamma_t_eigen* keywords are the eigenvalues of +the rotational and viscous damping tensors (having the same units as +their isotropic counterparts). Required for (and only compatible with) +*brownian/asphere*. For a 2D system, the first two values of *gamma_r_eigen* +must be inf (only rotation in xy plane), and the third value of *gamma_t_eigen* +must be inf (only diffusion in xy plane). + +If the *dipole* keyword is used, then the dipole moments of the particles +are updated as described above. Only compatible with *brownian/asphere* +(as *brownian/sphere* updates dipoles automatically). + +---------- + +.. note:: + For style *brownian/asphere*, the components *gamma_t_eigen* =(x,x,x) and + *gamma_r_eigen* = (y,y,y), the dynamics will replicate those of the + *brownian/sphere* style with *gamma_t* = x and *gamma_r* = y. + +---------- + +Restart, fix_modify, output, run start/stop, minimize info +""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +No information about this fix is written to :doc:`binary restart files `. +No global or per-atom quantities are stored +by this fix for access by various :doc:`output commands `. + + +No parameter of this fix can be used with the *start/stop* keywords of +the :doc:`run ` command. This fix is not invoked during +:doc:`energy minimization `. + +Restrictions +"""""""""""" + +The style *brownian/sphere* fix requires that atoms store torque and angular velocity (omega) +as defined by the :doc:`atom_style sphere ` command. +The style *brownian/asphere* fix requires that atoms store torque and quaternions +as defined by the :doc:`atom_style ellipsoid ` command. +If the *dipole* keyword is used, they must also store a dipole moment +as defined by the :doc:`atom_style dipole ` command. + +This fix is part of the USER-BROWNIAN package. It is only enabled if +LAMMPS was built with that package. See the :doc:`Build package ` +doc page for more info. + +Related commands +"""""""""""""""" + +:doc:`fix propel/self `, +:doc:`fix langevin `, :doc:`fix nve/sphere `, + +Default +""""""" + +The default for *rng* is *uniform*. The default for the rotational and translational friction +tensors are the identity tensor. + +---------- + +.. _Ilie1: + +**(Ilie)** Ilie, Briels, den Otter, Journal of Chemical Physics, 142, 114103 (2015). + +.. _Delong1: + +**(Delong)** Delong, Usabiaga, Donev, Journal of Chemical Physics. 143, 144107 (2015) + +.. _Dunweg7: + +**(Dunweg)** Dunweg and Paul, Int J of Modern Physics C, 2, 817-27 (1991). diff --git a/doc/src/fix_drude.rst b/doc/src/fix_drude.rst index 385a847749..a879f1594b 100644 --- a/doc/src/fix_drude.rst +++ b/doc/src/fix_drude.rst @@ -22,6 +22,8 @@ Examples fix 1 all drude 1 1 0 1 0 2 2 2 fix 1 all drude C C N C N D D D +Example input scripts available: examples/USER/drude + Description """"""""""" diff --git a/doc/src/fix_drude_transform.rst b/doc/src/fix_drude_transform.rst index 792b041576..28d57306ea 100644 --- a/doc/src/fix_drude_transform.rst +++ b/doc/src/fix_drude_transform.rst @@ -25,6 +25,8 @@ Examples fix 3 all drude/transform/direct fix 1 all drude/transform/inverse +Example input scripts available: examples/USER/drude + Description """"""""""" diff --git a/doc/src/fix_grem.rst b/doc/src/fix_grem.rst index e57f1889e1..85702b909f 100644 --- a/doc/src/fix_grem.rst +++ b/doc/src/fix_grem.rst @@ -92,6 +92,11 @@ The :doc:`thermo_modify ` *press* option is supported by this fix to add the rescaled kinetic pressure as part of :doc:`thermodynamic output `. +This fix computes a global scalar which can be accessed by various +:doc:`output commands `. The scalar is the effective +temperature :math:`T_{eff}`. The scalar value calculated by this +fix is "intensive". + Restrictions """""""""""" diff --git a/doc/src/fix_langevin_drude.rst b/doc/src/fix_langevin_drude.rst index ad5a876712..c075a56549 100644 --- a/doc/src/fix_langevin_drude.rst +++ b/doc/src/fix_langevin_drude.rst @@ -35,6 +35,8 @@ Examples fix 3 all langevin/drude 300.0 100.0 19377 1.0 20.0 83451 fix 1 all langevin/drude 298.15 100.0 19377 5.0 10.0 83451 zero yes +Example input scripts available: examples/USER/drude + Description """"""""""" diff --git a/doc/src/fix_mdi_engine.rst b/doc/src/fix_mdi_engine.rst new file mode 100644 index 0000000000..76506f3e19 --- /dev/null +++ b/doc/src/fix_mdi_engine.rst @@ -0,0 +1,59 @@ +.. index:: fix mdi/engine + +fix mdi/engine command +====================== + +Syntax +"""""" + +.. parsed-literal:: + + fix ID group-ID mdi/engine + +* ID, group-ID are documented in :doc:`fix ` command +* mdi/engine = style name of this fix command + +Examples +"""""""" + +.. code-block:: LAMMPS + + fix 1 all mdi/engine + +Description +""""""""""" + +This fix is used along with the :doc:`mdi/engine ` command +to enable LAMMPS to use the `MDI Library +`_ to run as +an MDI engine. The fix provides hooks that enable MDI driver codes to +communicate with LAMMPS at various points within a LAMMPS timestep. + +It is not generally necessary to add this fix to a LAMMPS input file, +even when using the :doc:`mdi/engine ` command. If the +:doc:`mdi/engine ` command is executed and this fix is not +present, it will automatically be added and applied as a new fix for +all atoms for the duration of the command. Thus it is only necessary +to add this fix to an input file when you want to modify the group-ID +or the ordering of this fix relative to other fixes in the input script. + +For more information about running LAMMPS as an MDI engine, see the +:doc:`mdi/engine ` command and the :doc:`Howto mdi +` doc page. + +Restrictions +"""""""""""" + +This command is part of the USER-MDI package. It is only enabled if +LAMMPS was built with that package. See the :doc:`Build package +` doc page for more info. + +Related commands +"""""""""""""""" + +:doc:`mdi/engine ` + +Default +""""""" + +none diff --git a/doc/src/fix_precession_spin.rst b/doc/src/fix_precession_spin.rst index 183753db5f..6a563bc32f 100644 --- a/doc/src/fix_precession_spin.rst +++ b/doc/src/fix_precession_spin.rst @@ -125,10 +125,10 @@ axis along the :math:`(1 1 1)`-type cube diagonals). :math:`K_2^c > diagonals. See chapter 2 of :ref:`(Skomski) ` for more details on cubic anisotropies. -Style *stt* is used to simulate the interaction between the spins and +Style *stt* is used to simulate the interaction between the spins and a spin-transfer torque. -See equation (7) of :ref:`(Chirac) ` for more details about the -implemented spin-transfer torque term. +See equation (7) of :ref:`(Chirac) ` for more details about the +implemented spin-transfer torque term. In all cases, the choice of :math:`(x y z)` only imposes the vector directions for the forces. Only the direction of the vector is @@ -139,12 +139,12 @@ Those styles can be combined within one single command line. .. note:: - The norm of all vectors defined with the precession/spin command + The norm of all vectors defined with the precession/spin command have to be non-zero. For example, defining "fix 1 all precession/spin zeeman 0.1 0.0 0.0 0.0" would result - in an error message. + in an error message. Since those vector components are used to compute the inverse of the - field (or anisotropy) vector norm, setting a zero-vector would result + field (or anisotropy) vector norm, setting a zero-vector would result in a division by zero. ---------- @@ -197,6 +197,6 @@ Oxford University Press. .. _Chirac1: -**(Chirac)** Chirac, Théophile, et al. Ultrafast antiferromagnetic -switching in NiO induced by spin transfer torques. +**(Chirac)** Chirac, Theophile, et al. Ultrafast antiferromagnetic +switching in NiO induced by spin transfer torques. Physical Review B 102.13 (2020): 134415. diff --git a/doc/src/fix_propel_self.rst b/doc/src/fix_propel_self.rst index 3a1bfb3166..ddc96817f9 100644 --- a/doc/src/fix_propel_self.rst +++ b/doc/src/fix_propel_self.rst @@ -8,52 +8,121 @@ Syntax .. parsed-literal:: - fix ID group-ID propel/self mode magnitude keyword values ... + fix ID group-ID propel/self mode magnitude keyword values * ID, group-ID are documented in :doc:`fix ` command * propel/self = style name of this fix command -* mode = velocity or quat -* magnitude = magnitude of the active force -* one or more keyword/value pairs may be appended to args -* keyword = *types* +* mode = *dipole* or *velocity* or *quat* +* magnitude = magnitude of self-propulsion force +* zero or one keyword/value pairs may be appended +* keyword = *qvector* + + .. parsed-literal:: + + *qvector* value = direction of force in ellipsoid frame + *sx*, *sy*, *sz* = components of *qvector* - *types* values = one or more atom types Examples """""""" .. code-block:: LAMMPS - fix active_group all propel/self velocity 1.0 - fix constant_velocity all viscous 1.0 - - fix active_group all propel/self quat 1.0 - - fix active all propel/self quat 1.0 types 1 2 4 + fix active all propel/self dipole 40.0 + fix active all propel/self velocity 10.0 + fix active all propel/self quat 15.7 qvector 1.0 0.0 0.0 Description """"""""""" -Adds a force of a constant magnitude to each atom in the group. The nature in -which the force is added depends on the mode. +Add a force to each atom in the group due to a self-propulsion force. The +force is given by -For *mode* = *velocity*, the active force acts along the velocity vector of -each atom. This can be interpreted as a velocity-dependent friction, -such as proposed by :ref:`(Erdmann) `. +.. math:: -For *mode* = *quat* the force is applied along the axis obtained -by rotating the x-axis along the atom's quaternion. In other words, the -force is along the x-axis in the atom's body frame. This mode requires -all atoms in the group to have a quaternion, so atom_style should -either be ellipsoid or body. In combination with Langevin thermostat -for translation and rotation in the overdamped regime, the quaternion -mode corresponds to the active Brownian particle model introduced by -:ref:`(Henkes) `, :ref:`(Bialke) ` and :ref:`(Fily) -`. + F_i = f_P e_i -By default, this fix is applied to all atoms in the group. You can -override this behavior by specifying the atom types the fix should work -on through the *types* keyword. +where *i* is the particle the force is being applied to, :math:`f_P` +is the magnitude of the force, and :math:`e_i` is the vector direction +of the force. The specification of :math:`e_i` is based on which of the +three keywords (*dipole* or *velocity* or *quat*) one selects. + +For mode *dipole*, :math:`e_i` is just equal to +the dipole vectors of the atoms in the group. Therefore, if the dipoles +are not unit vectors, the :math:`e_i` will not be unit vectors. + +.. note:: + + If another command changes the magnitude of the dipole, this force will + change accordingly (since :math:`|e_i|` will change, which is physically + equivalent to re-scaling :math:`f_P` while keeping :math:`|e_i|` constant), + and no warning will be provided by LAMMPS. This is almost never what you + want, so ensure you are not changing dipole magnitudes with another LAMMPS + fix or pair style. Furthermore, self-propulsion forces (almost) always + set :math:`e_i` to be a unit vector for all times, so it's best to set + all the dipole magnitudes to 1.0 unless you have a good reason not to + (see the :doc:`set ` command on how to do this). + +For mode *velocity*, :math:`e_i` points in the direction +of the current velocity (a unit-vector). This can be interpreted as a +velocity-dependent friction, as proposed by e.g. :ref:`(Erdmann) `. + +For mode *quat*, :math:`e_i` points in the direction of a unit +vector, oriented in the coordinate frame of the ellipsoidal particles, +which defaults to point along the x-direction. This default behavior +can be changed by via the *quatvec* keyword. + +The optional *quatvec* keyword specifies the direction of self-propulsion +via a unit vector (sx,sy,sz). The arguments *sx*, *sy*, and *sz*, are +defined within the coordinate frame of the atom's +ellipsoid. For instance, for an ellipsoid with long axis along +its x-direction, if one wanted the self-propulsion force to also +be along this axis, set *sx* equal to 1 and *sy*, *sz* both equal +to zero. This keyword may only be specified for mode *quat*. + +.. note:: + + In using keyword *quatvec*, the three arguments *sx*, + *sy*, and *sz* will be automatically normalized to components + of a unit vector internally to avoid users having to explicitly + do so themselves. Therefore, in mode *quat*, the vectors :math:`e_i` + will always be of unit length. + + +Along with adding a force contribution, this fix can also +contribute to the virial (pressure) of the system, defined as +:math:`f_P \sum_i /(d V)`, where :math:`r_i` is the +*unwrapped* coordinate of particle i in the case of periodic +boundary conditions. See :ref:`(Winkler) ` for a +discussion of this active pressure contribution. + +For modes *dipole* and *quat*, this fix is by default +included in pressure computations. + +For mode *velocity*, this fix is by default not included +in pressure computations. + + +.. note:: + + In contrast to equilibrium systems, pressure of active systems + in general depends on the geometry of the container. + The active pressure contribution as calculated in this fix + is only valid for certain boundary conditions (spherical + walls, rectangular walls, or periodic boundary conditions). + For other geometries, the pressure must be measured via + explicit calculation of the force per unit area on a wall, + and so one must not calculate it using this fix. + (Use :doc:`fix_modify ` as described below + to turn off the virial contribution of this fix). Again, + see :ref:`(Winkler) ` for discussion of why this + is the case. + + Furthermore, when dealing with active systems, the temperature + is no longer well defined. Therefore, one should ensure that + the *virial* flag is used in the + :doc:`compute pressure ` command (turning + off temperature contributions). ---------- @@ -62,40 +131,48 @@ Restart, fix_modify, output, run start/stop, minimize info No information about this fix is written to :doc:`binary restart files `. -This fix is not imposed during minimization. +The :doc:`fix_modify ` *virial* option is supported by this +fix to add the contribution due to the added forces on atoms to the +system's virial as part of :doc:`thermodynamic output `. +The default is *virial yes* for keywords *dipole* and *quat*. The +default is *virial no* for keyword *velocity*. + + +No parameter of this fix can be used with the *start/stop* keywords of +the :doc:`run ` command. + Restrictions """""""""""" -In quat mode, this fix makes use of per-atom quaternions to take -into account the fact that the orientation can rotate and hence the -direction of the active force can change. The quat mode -of this fix only works with atom_style ellipsoid. +With keyword *dipole*, this fix only works when the DIPOLE package is enabled. +See the :doc:`Build package ` doc page for more info. + +This fix is part of the USER-BROWNIAN package. It is only enabled if +LAMMPS was built with that package. See the :doc:`Build package ` +doc page for more info. + Related commands """""""""""""""" -:doc:`fix setforce `, :doc:`fix addforce ` - -.. _Erdmann: - -**(Erdmann)** U. Erdmann , W. Ebeling, L. Schimansky-Geier, and F. Schweitzer, -Eur. Phys. J. B 15, 105-113, 2000. - -.. _Henkes: - -**(Henkes)** Henkes, S, Fily, Y., and Marchetti, M. C. Phys. Rev. E, 84, 040301(R), 2011. - -.. _Bialke: - -**(Bialke)** J. Bialke, T. Speck, and H Loewen, Phys. Rev. Lett. 108, 168301, 2012. - -.. _Fily: - -**(Fily)** Y. Fily and M.C. Marchetti, Phys. Rev. Lett. 108, 235702, 2012. +:doc:`fix efield ` , :doc:`fix setforce `, +:doc:`fix addforce ` Default """"""" -types +none +---------- + + +.. _Erdmann1: + +**(Erdmann)** U. Erdmann , W. Ebeling, L. Schimansky-Geier, and F. Schweitzer, +Eur. Phys. J. B 15, 105-113, 2000. + + +.. _Winkler1: + +**(Winkler)** Winkler, Wysocki, and Gompper, Soft Matter, 11, 6680 (2015). diff --git a/doc/src/fix_tgnh_drude.rst b/doc/src/fix_tgnh_drude.rst index 1854655a82..a6e9832833 100644 --- a/doc/src/fix_tgnh_drude.rst +++ b/doc/src/fix_tgnh_drude.rst @@ -62,6 +62,8 @@ Examples fix 2 jello tgnpt/drude temp 300.0 300.0 100.0 1.0 20.0 tri 5.0 5.0 1000.0 fix 2 ice tgnpt/drude temp 250.0 250.0 100.0 1.0 20.0 x 1.0 1.0 0.5 y 2.0 2.0 0.5 z 3.0 3.0 0.5 yz 0.1 0.1 0.5 xz 0.2 0.2 0.5 xy 0.3 0.3 0.5 nreset 1000 +Example input scripts available: examples/USER/drude + Description """"""""""" diff --git a/doc/src/mdi_engine.rst b/doc/src/mdi_engine.rst new file mode 100644 index 0000000000..b39999606d --- /dev/null +++ b/doc/src/mdi_engine.rst @@ -0,0 +1,88 @@ +.. index:: mdi/engine + +mdi_engine command +================== + +Syntax +"""""" + +.. parsed-literal:: + + mdi_engine + +Description +""""""""""" + +This command is used to have LAMMPS act as a server with another +client code to effectively couple the two codes together in +client/server mode. + +More specifically, this command causes LAMMPS to begin using the `MDI +Library `_ +to run as an MDI engine (server), responding to commands made by an +external MDI driver code (client). See the :doc:`Howto mdi +` doc page for more information about how LAMMPS can work +as both an MDI driver or engine. + +General information about launching codes that communicate using the +MDI Library can be found in the `corresponding page +`_ +of the MDI Library's documentation. + +---------- + +This command should typically be used in an input script after LAMMPS +has setup the system it is going to model in collaboration with the +driver code. Depending on how the driver code tells the LAMMPS engine +to exit, other commands can be executed after this command, but +typically it should be used at the end of the LAMMPS input script. + +To act as a MD-based MDI engine, this is the list of MDI commands from +a driver code which LAMMPS currently recognizes. See more details +about these commands in the `MDI library documentation +`_ +.. NOTE: Taylor - is this the best link for this info? Can we flesh this +.. out with the full list of supported commands? Maybe the distinction +.. of what "node" the commands refer to is not needed in this table? + +.. list-table:: + :widths: 20 80 + :header-rows: 1 + + * - Command name + - Action + * - >NATOMS + - Driver sends the number of atoms in the system + * - FORCES + - Driver sends 3*N double-precision atom forces + * - ` doc page for more info. + +Related commands +"""""""""""""""" + +:doc:`fix mdi/engine ` + +Default +""""""" + +None diff --git a/doc/src/neigh_modify.rst b/doc/src/neigh_modify.rst index 2618953dd7..25c2c2ac77 100644 --- a/doc/src/neigh_modify.rst +++ b/doc/src/neigh_modify.rst @@ -14,7 +14,7 @@ Syntax .. parsed-literal:: - keyword = *delay* or *every* or *check* or *once* or *cluster* or *include* or *exclude* or *page* or *one* or *binsize* + keyword = *delay* or *every* or *check* or *once* or *cluster* or *include* or *exclude* or *page* or *one* or *binsize* or *collection/type* or *collection/interval* *delay* value = N N = delay building until this many steps since last build *every* value = M @@ -47,6 +47,12 @@ Syntax N = max number of neighbors of one atom *binsize* value = size size = bin size for neighbor list construction (distance units) + *collection/type* values = N arg1 ... argN + N = number of custom collections + arg = N separate lists of types (see below) + *collection/interval* values = N arg1 ... argN + N = number of custom collections + arg = N separate cutoffs for intervals (see below) Examples """""""" @@ -58,6 +64,8 @@ Examples neigh_modify exclude group frozen frozen check no neigh_modify exclude group residue1 chain3 neigh_modify exclude molecule/intra rigid + neigh_modify collection/type 2 1*2,5 3*4 + neigh_modify collection/interval 2 1.0 10.0 Description """"""""""" @@ -188,8 +196,9 @@ atom can have. The *binsize* option allows you to specify what size of bins will be used in neighbor list construction to sort and find neighboring atoms. By default, for :doc:`neighbor style bin `, LAMMPS uses bins -that are 1/2 the size of the maximum pair cutoff. For :doc:`neighbor style multi `, the bins are 1/2 the size of the minimum pair -cutoff. Typically these are good values for minimizing the time for +that are 1/2 the size of the maximum pair cutoff. For :doc:`neighbor style multi `, +the bins are 1/2 the size of the collection interaction cutoff. +Typically these are good values for minimizing the time for neighbor list construction. This setting overrides the default. If you make it too big, there is little overhead due to looping over bins, but more atoms are checked. If you make it too @@ -197,6 +206,31 @@ small, the optimal number of atoms is checked, but bin overhead goes up. If you set the binsize to 0.0, LAMMPS will use the default binsize of 1/2 the cutoff. +The *collection/type* option allows you to define collections of atom +types, used by the *multi* neighbor mode. By grouping atom types with +similar physical size or interaction cutoff lengths, one may be able +to improve performance by reducing +overhead. You must first specify the number of collections N to be +defined followed by N lists of types. Each list consists of a series of type +ranges separated by commas. The range can be specified as a +single numeric value, or a wildcard asterisk can be used to specify a range +of values. This takes the form "\*" or "\*n" or "n\*" or "m\*n". For +example, if M = the number of atom types, then an asterisk with no numeric +values means all types from 1 to M. A leading asterisk means all types +from 1 to n (inclusive). A trailing asterisk means all types from n to M +(inclusive). A middle asterisk means all types from m to n (inclusive). +Note that all atom types must be included in exactly one of the N collections. + +The *collection/interval* option provides a similar capability. This +command allows a user to define collections by specifying a series of +cutoff intervals. LAMMPS will automatically sort atoms into these +intervals based on their type-dependent cutoffs or their finite size. +You must first specify the number of collections N to be defined +followed by N values representing the upper cutoff of each interval. +This command is particularly useful for granular pair styles where the +interaction distance of particles depends on their radius and may not +depend on their atom type. + Restrictions """""""""""" diff --git a/doc/src/neighbor.rst b/doc/src/neighbor.rst index 98ee0d0b6a..1b10ec8998 100644 --- a/doc/src/neighbor.rst +++ b/doc/src/neighbor.rst @@ -11,7 +11,7 @@ Syntax neighbor skin style * skin = extra distance beyond force cutoff (distance units) -* style = *bin* or *nsq* or *multi* +* style = *bin* or *nsq* or *multi* or *multi/old* Examples """""""" @@ -55,14 +55,31 @@ For the *bin* style, the bin size is set to 1/2 of the largest cutoff distance between any pair of atom types and a single set of bins is defined to search over for all atom types. This can be inefficient if one pair of types has a very long cutoff, but -other type pairs have a much shorter cutoff. For style *multi* the -bin size is set to 1/2 of the shortest cutoff distance and multiple -sets of bins are defined to search over for different atom types. +other type pairs have a much shorter cutoff. The *multi* style uses +different sized bins for collections of different sized particles, where +"size" may mean the physical size of the particle or its cutoff +distance for interacting with other particles. Different +sets of bins are then used to construct the neighbor lists as as further +described by Shire, Hanley, and Stratford :ref:`(Shire) `. This imposes some extra setup overhead, but the searches themselves -may be much faster for the short-cutoff cases. -See the :doc:`comm_modify mode multi ` command for a -communication option that may also be beneficial for simulations of -this kind. +may be much faster. By default, each atom type defines a separate +collection of particles. For systems where two or more atom types +have the same size (either physical size or cutoff distance), the +definition of collections can be customized, which can result in less +overhead and faster performance. See the :doc:`neigh_modify ` +command for how to define custom collections. Whether the collection +definition is customized or not, also see the +:doc:`comm_modify mode multi ` command for communication +options that further improve performance in a manner consistent with +neighbor style multi. + +An alternate style, *multi/old*, sets the bin size to 1/2 of the shortest +cutoff distance and multiple sets of bins are defined to search over for +different atom types. This algorithm used to be the default *multi* +algorithm in LAMMPS but was found to be significantly slower than the new +approach. For now we are keeping the old option in case there are use cases +where multi/old outperforms the new multi style. + The :doc:`neigh_modify ` command has additional options that control how often neighbor lists are built and which pairs are @@ -90,3 +107,9 @@ Default | 0.001 bin for units = si, skin = 0.001 meters = 1.0 mm | 0.1 bin for units = cgs, skin = 0.1 cm = 1.0 mm | + +---------- + +.. _bytype-Shire: + +**(Shire)** Shire, Hanley and Stratford, Comp Part Mech, (2020). diff --git a/doc/src/pair_coul_tt.rst b/doc/src/pair_coul_tt.rst index 11b4e72a60..b01d46d7ae 100644 --- a/doc/src/pair_coul_tt.rst +++ b/doc/src/pair_coul_tt.rst @@ -29,6 +29,7 @@ Examples pair_coeff 1 2 coul/tt 4.0 1.0 4 12.0 pair_coeff 1 3* coul/tt 4.5 1.0 4 +Example input scripts available: examples/USER/drude Description """"""""""" diff --git a/doc/src/pair_dpd_ext.rst b/doc/src/pair_dpd_ext.rst new file mode 100644 index 0000000000..cfc72fe84c --- /dev/null +++ b/doc/src/pair_dpd_ext.rst @@ -0,0 +1,138 @@ +.. index:: pair_style dpd/ext +.. index:: pair_style dpd/ext/tstat + +pair_style dpd/ext command +========================== + +pair_style dpd/ext/tstat command +================================ + +Syntax +"""""" + + +.. code-block:: LAMMPS + + pair_style dpd/ext T cutoff seed + pair_style dpd/ext/tstat Tstart Tstop cutoff seed + +* T = temperature (temperature units) +* Tstart,Tstop = desired temperature at start/end of run (temperature units) +* cutoff = global cutoff for DPD interactions (distance units) +* seed = random # seed (positive integer) + +Examples +"""""""" + + +.. code-block:: LAMMPS + + pair_style dpd/ext 1.0 2.5 34387 + pair_coeff 1 1 25.0 4.5 4.5 0.5 0.5 1.2 + pair_coeff 1 2 40.0 4.5 4.5 0.5 0.5 1.2 + + pair_style dpd/ext/tstat 1.0 1.0 2.5 34387 + pair_coeff 1 1 4.5 4.5 0.5 0.5 1.2 + pair_coeff 1 2 4.5 4.5 0.5 0.5 1.2 + +Description +""""""""""" + +The style *dpd/ext* computes an extended force field for dissipative particle dynamics (DPD) following the exposition in :ref:`(Groot) `, :ref:`(Junghans) `. + +Style *dpd/ext/tstat* invokes an extended DPD thermostat on pairwise interactions, equivalent to the non-conservative portion of the extended DPD force field. To use *dpd/ext/tstat* as a thermostat for another pair style, use the :doc:`pair_style hybrid/overlay ` command to compute both the desired pair interaction and the thermostat for each pair of particles. + +For the style *dpd/ext*\ , the force on atom I due to atom J is given as a sum +of 3 terms + +.. math:: + + \mathbf{f} = & f^C + f^D + f^R \qquad \qquad r < r_c \\ + f^C = & A_{ij} w(r) \hat{\mathbf{r}}_{ij} \\ + f^D = & - \gamma_{\parallel} w_{\parallel}^2(r) (\hat{\mathbf{r}}_{ij} \cdot \mathbf{v}_{ij}) \hat{\mathbf{r}}_{ij} - \gamma_{\perp} w_{\perp}^2 (r) ( \mathbf{I} - \hat{\mathbf{r}}_{ij} \hat{\mathbf{r}}_{ij}^{\rm T} ) \mathbf{v}_{ij} \\ + f^R = & \sigma_{\parallel} w_{\parallel}(r) \frac{\alpha}{\sqrt{\Delta t}} \hat{\mathbf{r}}_{ij} + \sigma_{\perp} w_{\perp} (r) ( \mathbf{I} - \hat{\mathbf{r}}_{ij} \hat{\mathbf{r}}_{ij}^{\rm T} ) \frac{\mathbf{\xi}_{ij}}{\sqrt{\Delta t}}\\ + w(r) = & 1 - r/r_c \\ + +where :math:`\mathbf{f}^C` is a conservative force, :math:`\mathbf{f}^D` is a dissipative force, and :math:`\mathbf{f}^R` is a random force. :math:`A_{ij}` is the maximum repulsion between the two atoms, :math:`\hat{\mathbf{r}}_{ij}` is a unit vector in the direction :math:`\mathbf{r}_i - \mathbf{r}_j`, :math:`\mathbf{v}_{ij} = \mathbf{v}_i - \mathbf{v}_j` is the vector difference in velocities of the two atoms, :math:`\alpha` and :math:`\mathbf{\xi}_{ij}` are Gaussian random numbers with zero mean and unit variance, :math:`\Delta t` is the timestep, :math:`w (r) = 1 - r / r_c` is a weight function for the conservative interactions that varies between 0 and 1, :math:`r_c` is the corresponding cutoff, :math:`w_{\alpha} ( r ) = ( 1 - r / \bar{r}_c )^{s_{\alpha}}`, :math:`\alpha \equiv ( \parallel, \perp )`, are weight functions with coefficients :math:`s_\alpha` that vary between 0 and 1, :math:`\bar{r}_c` is the corresponding cutoff, :math:`\mathbf{I}` is the unit matrix, :math:`\sigma_{\alpha} = \sqrt{2 k T \gamma_{\alpha}}`, where :math:`k` is the Boltzmann constant and :math:`T` is the temperature in the pair\_style command. + +For the style *dpd/ext/tstat*\ , the force on atom I due to atom J is the same as the above equation, except that the conservative :math:`\mathbf{f}^C` term is dropped. Also, during the run, T is set each timestep to a ramped value from Tstart to Tstop. + +For the style *dpd/ext*\ , the pairwise energy associated with style *dpd/ext* is only due to the conservative force term :math:`\mathbf{f}^C`, and is shifted to be zero at the cutoff distance :math:`r_c`. The pairwise virial is calculated using all three terms. There is no pairwise energy for style *dpd/ext/tstat*, but the last two terms of the formula contribute the virial. + +For the style *dpd/ext/tstat*, the force on atom I due to atom J is the same as the above equation, except that the conservative :math:`\mathbf{f}^C` term is dropped. Also, during the run, T is set each timestep to a ramped value from Tstart to Tstop. + +For the style *dpd/ext*\ , the pairwise energy associated with style *dpd/ext* is only due to the conservative force term :math:`\mathbf{f}^C`, and is shifted to be zero at the cutoff distance :math:`r_c`. The pairwise virial is calculated using all three terms. There is no pairwise energy for style *dpd/ext/tstat*, but the last two terms of the formula contribute the virial. + +For the style *dpd/ext*, the following coefficients must be defined for each pair of atoms types via the :doc:`pair_coeff ` command as in the examples above: + +* A (force units) +* :math:`\gamma_{\perp}` (force/velocity units) +* :math:`\gamma_{\parallel}` (force/velocity units) +* :math:`s_{\perp}` (unitless) +* :math:`s_{\parallel}` (unitless) +* :math:`r_c` (distance units) + +The last coefficient is optional. If not specified, the global DPD cutoff is used. Note that :math:`\sigma`'s are set equal to :math:`\sqrt{2 k T \gamma}`, where :math:`T` is the temperature set by the :doc:`pair_style ` command so it does not need to be specified. + + +For the style *dpd/ext/tstat*, the coefficients defined for each pair of atoms types via the :doc:`pair_coeff ` command is the same, except that A is not included. + +.. note:: + + If you are modeling DPD polymer chains, you may want to use the :doc:`pair_style srp ` command in conjunction with these pair styles. It is a soft segmental repulsive potential (SRP) that can prevent DPD polymer chains from crossing each other. + +.. note:: + + The virial calculation for pressure when using this pair style includes all the components of force listed above, including the random force. + +---------- + + +**Mixing, shift, table, tail correction, restart, rRESPA info**\ : + +The style *dpd/ext* does not support mixing. Thus, coefficients for all I,J pairs must be specified explicitly. + +The pair styles do not support the :doc:`pair_modify ` shift option for the energy of the pair interaction. Note that as discussed above, the energy due to the conservative :math:`\mathbf{f}^C` term is already shifted to be zero at the cutoff distance :math:`r_c`. + +The :doc:`pair_modify ` table option is not relevant for the style *dpd/ext*. + +The style *dpd/ext* does not support the :doc:`pair_modify ` tail option for adding long-range tail corrections to energy and pressure. + +The pair styles can only be used via the pair keyword of the :doc:`run_style respa ` command. They do not support the *inner*\ , *middle*\ , and *outer*\ keywords. + +The style *dpd/ext/tstat* can ramp its target temperature over multiple runs, using the start and stop keywords of the :doc:`run ` command. See the :doc:`run ` command for details of how to do this. + +---------- + + +Restrictions +"""""""""""" + +The default frequency for rebuilding neighbor lists is every 10 steps (see the :doc:`neigh_modify ` command). This may be too infrequent for style *dpd/ext* simulations since particles move rapidly and can overlap by large amounts. If this setting yields a non-zero number of \say{dangerous} reneighborings (printed at the end of a simulation), you should experiment with forcing reneighboring more often and see if system energies/trajectories change. + +The pair styles require to use the :doc:`comm_modify vel yes ` command so that velocities are stored by ghost atoms. + +The pair styles will not restart exactly when using the :doc:`read_restart ` command, though they should provide statistically similar results. This is because the forces they compute depend on atom velocities. See the :doc:`read_restart ` command for more details. + +Related commands +"""""""""""""""" + +:doc:`pair_style dpd `, :doc:`pair_coeff `, :doc:`fix nvt `, :doc:`fix langevin `, :doc:`pair_style srp ` + +**Default:** none + + +---------- + + +.. _Groot: + + + +**(Groot)** Groot and Warren, J Chem Phys, 107, 4423-35 (1997). + +.. _Junghans: + + + +**(Junghans)** Junghans, Praprotnik and Kremer, Soft Matter 4, 156, 1119-1128 (2008). diff --git a/doc/src/pair_fep_soft.rst b/doc/src/pair_fep_soft.rst index 1bb50fad1f..ce8227abeb 100644 --- a/doc/src/pair_fep_soft.rst +++ b/doc/src/pair_fep_soft.rst @@ -182,6 +182,8 @@ Examples pair_coeff * * 100.0 2.0 1.5 1.0 pair_coeff 1 1 100.0 2.0 1.5 1.0 3.0 +Example input scripts available: examples/USER/fep + Description """"""""""" diff --git a/doc/src/pair_hybrid.rst b/doc/src/pair_hybrid.rst index 868d645b83..f7de713c94 100644 --- a/doc/src/pair_hybrid.rst +++ b/doc/src/pair_hybrid.rst @@ -257,7 +257,7 @@ then overrides them with 0.0 only for CHARMM: .. code-block:: LAMMPS special_bonds amber - pair_hybrid lj/charmm/coul/long 8.0 10.0 lj/cut/coul/long 10.0 + pair_style hybrid lj/charmm/coul/long 8.0 10.0 lj/cut/coul/long 10.0 pair_modify pair lj/charmm/coul/long special lj/coul 0.0 0.0 0.0 The this input achieves the same effect: @@ -265,7 +265,7 @@ The this input achieves the same effect: .. code-block:: LAMMPS special_bonds 0.0 0.0 0.1 - pair_hybrid lj/charmm/coul/long 8.0 10.0 lj/cut/coul/long 10.0 + pair_style hybrid lj/charmm/coul/long 8.0 10.0 lj/cut/coul/long 10.0 pair_modify pair lj/cut/coul/long special lj 0.0 0.0 0.5 pair_modify pair lj/cut/coul/long special coul 0.0 0.0 0.83333333 pair_modify pair lj/charmm/coul/long special lj/coul 0.0 0.0 0.0 @@ -279,7 +279,7 @@ effectively *lj/coul 0.0 0.0 0.5* as required for OPLS/AA: .. code-block:: LAMMPS special_bonds lj/coul 1e-20 1e-20 0.5 - pair_hybrid tersoff lj/cut/coul/long 12.0 + pair_style hybrid tersoff lj/cut/coul/long 12.0 pair_modify pair tersoff special lj/coul 1.0 1.0 1.0 For use with the various :doc:`compute \*/tally ` @@ -417,7 +417,11 @@ assigned automatically to the sub-style defined for both I,I and J,J and its coefficients generated by the mixing rule used by that sub-style. For the *hybrid/overlay* and *hybrid/scaled* style, there is an additional requirement that both the I,I and J,J pairs are assigned to a -single sub-style. See the :doc:`pair_modify ` command for +single sub-style. If this requirement is not met, no I,J coeffs will be +generated, even if the sub-styles support mixing, and I,J pair +coefficients must be explicitly defined. + +See the :doc:`pair_modify ` command for details of mixing rules. See the See the doc page for the sub-style to see if allows for mixing. diff --git a/doc/src/pair_lj_cut_tip4p.rst b/doc/src/pair_lj_cut_tip4p.rst index 4f55f3b14e..5f3b4e48e7 100644 --- a/doc/src/pair_lj_cut_tip4p.rst +++ b/doc/src/pair_lj_cut_tip4p.rst @@ -145,6 +145,22 @@ specified since a Coulombic cutoff cannot be specified for an individual I,J type pair. All type pairs use the same global Coulombic cutoff specified in the pair_style command. +.. warning:: + + Because of how these pair styles implement the coulomb interactions + by implicitly defining a fourth site for the negative charge + of the TIP4P and similar water models, special care must be taken + when using these pair styles with other computations that also use + charges. Unless they are specially set up to also handle the implicit + definition of the 4th site, results are likely incorrect. Example: + :doc:`compute dipole/chunk `. For the same + reason, when using one of these pair styles with + :doc:`pair_style hybrid `, **all** coulomb interactions + should be handled by a single sub-style with TIP4P support. All other + instances and styles will "see" the M point charges at the position + of the Oxygen atom and thus compute incorrect forces and energies. + LAMMPS will print a warning when it detects one of these issues. + ---------- A version of these styles with a soft core, *lj/cut/tip4p/long/soft*\ , suitable diff --git a/doc/src/pair_lj_smooth.rst b/doc/src/pair_lj_smooth.rst index 7ba12c89ba..a77eefd023 100644 --- a/doc/src/pair_lj_smooth.rst +++ b/doc/src/pair_lj_smooth.rst @@ -1,10 +1,11 @@ .. index:: pair_style lj/smooth +.. index:: pair_style lj/smooth/gpu .. index:: pair_style lj/smooth/omp pair_style lj/smooth command ============================ -Accelerator Variants: *lj/smooth/omp* +Accelerator Variants: *lj/smooth/gpu*, *lj/smooth/omp* Syntax """""" diff --git a/doc/src/pair_style.rst b/doc/src/pair_style.rst index 0adc6c6e48..d77627786d 100644 --- a/doc/src/pair_style.rst +++ b/doc/src/pair_style.rst @@ -150,6 +150,8 @@ accelerated styles exist. * :doc:`coul/wolf ` - Coulomb via Wolf potential * :doc:`coul/wolf/cs ` - Coulomb via Wolf potential with core/shell adjustments * :doc:`dpd ` - dissipative particle dynamics (DPD) +* :doc:`dpd/ext ` - generalized force field for DPD +* :doc:`dpd/ext/tstat ` - pair-wise DPD thermostatting with generalized force field * :doc:`dpd/fdt ` - DPD for constant temperature and pressure * :doc:`dpd/fdt/energy ` - DPD for constant energy and enthalpy * :doc:`dpd/tstat ` - pair-wise DPD thermostatting diff --git a/doc/src/pair_thole.rst b/doc/src/pair_thole.rst index a8cf8ee044..183bab48c9 100644 --- a/doc/src/pair_thole.rst +++ b/doc/src/pair_thole.rst @@ -42,6 +42,8 @@ Examples pair_style lj/cut/thole/long 2.6 12.0 +Example input scripts available: examples/USER/drude + Description """"""""""" diff --git a/doc/src/read_data.rst b/doc/src/read_data.rst index 5d74d9c28b..bd7c480af2 100644 --- a/doc/src/read_data.rst +++ b/doc/src/read_data.rst @@ -223,6 +223,9 @@ The structure of the data file is important, though many settings and sections are optional or can come in any order. See the examples directory for sample data files for different problems. +The file will be read line by line, but there is a limit of 254 +characters per line and characters beyond that limit will be ignored. + A data file has a header and a body. The header appears first. The first line of the header is always skipped; it typically contains a description of the file. Then lines are read one at a time. Lines diff --git a/doc/src/thermo_modify.rst b/doc/src/thermo_modify.rst index 4439f7732c..1815a7cb76 100644 --- a/doc/src/thermo_modify.rst +++ b/doc/src/thermo_modify.rst @@ -11,12 +11,13 @@ Syntax thermo_modify keyword value ... * one or more keyword/value pairs may be listed -* keyword = *lost* or *lost/bond* or *norm* or *flush* or *line* or *format* or *temp* or *press* +* keyword = *lost* or *lost/bond* or *warn* or *norm* or *flush* or *line* or *format* or *temp* or *press* .. parsed-literal:: *lost* value = *error* or *warn* or *ignore* *lost/bond* value = *error* or *warn* or *ignore* + *warn* value = *ignore* or *reset* or *default* or a number *norm* value = *yes* or *no* *flush* value = *yes* or *no* *line* value = *one* or *multi* @@ -75,6 +76,43 @@ are drifting out of the box through a fixed boundary condition (see the :doc:`boundary ` command). In this case one atom may be deleted before the rest of the molecule is, on a later timestep. +The *warn* keyword allows you to control whether LAMMPS will print +warning messages and how many of them. Most warning messages are only +printed by MPI rank 0. They are usually pointing out important issues +that should be investigated, but LAMMPS cannot determine for +certain whether they are an indication of an error. + +Some warning messages are printed during a run (or immediately before) +each time a specific MPI rank encounters the issue, e.g. bonds that are +stretched too far or dihedrals in extreme configurations. These number +of these can quickly blow up the size of the log file and screen output. +Thus a limit of 100 warning messages is applied by default. The warning +count is applied to the entire input unless reset with a ``thermo_modify +warn reset`` command. If there are more warnings than the limit, LAMMPS +will print one final warning that it will not print any additional +warning messages. + +.. note:: + + The warning limit is enforced on either the per-processor count or + the total count across all processors. For efficiency reasons, + however, the total count is only updated at steps with thermodynamic + output. Thus when running on a large number of processors in + parallel, the total number of warnings printed can be significantly + larger than the given limit. + +Any number after the keyword *warn* will change the warning limit +accordingly. With the value *ignore* all warnings will be suppressed, +with the value *always* no limit will be applied and warnings will +always be printed, with the value *reset* the internal warning counter +will be reset to zero, and with the value *default*, the counter is +reset and the limit set to 100. An example usage of either *reset* or +*default* would be to re-enable warnings that were disabled or have +reached the limit during equilibration, where the warnings would be +acceptable while the system is still adjusting, but then change +to all warnings for the production run, where they would indicate +problems that would require a closer look at what is causing them. + The *norm* keyword determines whether various thermodynamic output values are normalized by the number of atoms or not, depending on whether it is set to *yes* or *no*\ . Different unit styles have @@ -183,9 +221,9 @@ Related commands Default """"""" -The option defaults are lost = error, norm = yes for unit style of -*lj*\ , norm = no for unit style of *real* and *metal*\ , flush = no, -and temp/press = compute IDs defined by thermo_style. +The option defaults are lost = error, warn = 100, norm = yes for unit +style of *lj*\ , norm = no for unit style of *real* and *metal*\ , +flush = no, and temp/press = compute IDs defined by thermo_style. The defaults for the line and format options depend on the thermo style. For styles "one" and "custom", the line and format defaults diff --git a/doc/src/variable.rst b/doc/src/variable.rst index af324c180f..4747c492aa 100644 --- a/doc/src/variable.rst +++ b/doc/src/variable.rst @@ -363,12 +363,14 @@ variable, as discussed below. The rules for formatting the file are as follows. Each time a set of per-atom values is read, a non-blank line is searched for in the file. -A comment character "#" can be used anywhere on a line; text starting -with the comment character is stripped. Blank lines are skipped. The -first "word" of a non-blank line, delimited by white-space, is read as -the count N of per-atom lines to immediately follow. N can be the -total number of atoms in the system, or only a subset. The next N -lines have the following format +The file is read line by line but only up to 254 characters are used. +The rest are ignored. A comment character "#" can be used anywhere +on a line and all text following and the "#" character are ignored; +text starting with the comment character is stripped. Blank lines +are skipped. The first "word" of a non-blank line, delimited by +white-space, is read as the count N of per-atom lines to immediately +follow. N can be the total number of atoms in the system, or only a +subset. The next N lines have the following format .. parsed-literal:: diff --git a/doc/utils/requirements.txt b/doc/utils/requirements.txt index 00fa6ecfaf..9b8e106875 100644 --- a/doc/utils/requirements.txt +++ b/doc/utils/requirements.txt @@ -4,3 +4,4 @@ git+git://github.com/akohlmey/sphinx-fortran@parallel-read sphinx_tabs breathe Pygments +six diff --git a/doc/utils/sphinx-config/_static/css/lammps.css b/doc/utils/sphinx-config/_static/css/lammps.css index b69405365a..900054eeeb 100644 --- a/doc/utils/sphinx-config/_static/css/lammps.css +++ b/doc/utils/sphinx-config/_static/css/lammps.css @@ -35,6 +35,10 @@ hr { display: none; } +#command-reference.section h2 { + display: none; +} + .ui.tabular.menu .item { padding-right: 1em; padding-left: 1em; diff --git a/doc/utils/sphinx-config/conf.py.in b/doc/utils/sphinx-config/conf.py.in index 17cb3c2395..baf472b9f9 100644 --- a/doc/utils/sphinx-config/conf.py.in +++ b/doc/utils/sphinx-config/conf.py.in @@ -241,7 +241,7 @@ html_show_sourcelink = False # Output file base name for HTML help builder. htmlhelp_basename = 'LAMMPSdoc' -html_add_permalinks = '' +html_permalinks = True if 'epub' in sys.argv: html_math_renderer = 'imgmath' diff --git a/doc/utils/sphinx-config/false_positives.txt b/doc/utils/sphinx-config/false_positives.txt index da0617ce91..6133729b8a 100644 --- a/doc/utils/sphinx-config/false_positives.txt +++ b/doc/utils/sphinx-config/false_positives.txt @@ -47,6 +47,7 @@ Agnolin Ai Aidan aij +aimd airebo Aj ajs @@ -101,6 +102,7 @@ anisotropic anisotropies anisotropy ansi +antiferromagnetic antiquewhite Antisymmetrized antisymmetry @@ -195,6 +197,7 @@ Ballenegger Bammann Banna Barashev +barnes barostat Barostats barostatted @@ -311,6 +314,7 @@ br Branduardi Branicio brennan +Briels Brien Brilliantov Broadwell @@ -625,6 +629,7 @@ delflag Dellago delocalization delocalized +Delong delr deltaHf Dendrimer @@ -708,6 +713,7 @@ dodgerblue dof doi Donadio +Donev dotc Doty doxygen @@ -1023,6 +1029,7 @@ fmag fmass fmm fmt +fmtlib fmx fmy fmz @@ -1141,6 +1148,7 @@ GMock gneb GNEB Goldfarb +Gompper Gonzalez-Melchor googlemail googletest @@ -1201,6 +1209,7 @@ Halver Hamaker Hamel Hammerschmidt +Hanley haptic Hara Harpertown @@ -1263,6 +1272,7 @@ holonomic Homebrew hooke Hookean +hostname hotpink Houlle howto @@ -1319,6 +1329,7 @@ ijk ijkl ik Ikeshoji +Ilie ilmenau Ilmenau ilp @@ -1721,6 +1732,7 @@ Lmpsdata lmptype LMT ln +localhost localTemp localvectors Loewen @@ -1851,6 +1863,8 @@ mc McLachlan md mdf +MDI +mdi mdpd mDPD meam @@ -2177,6 +2191,7 @@ nimpropers nimpropertypes Nimpropertype Ninteger +NiO Nissila nist nitride @@ -2188,6 +2203,7 @@ nl nlayers nlen Nlines +nlines nlo nlocal Nlocal @@ -2279,6 +2295,7 @@ Ntype ntypes Ntypes nucleotides +nullptr num numa numactl @@ -2538,6 +2555,7 @@ ppn pppm prd Prakash +Praprotnik pre Pre prec @@ -2832,6 +2850,8 @@ Runge runtime Rutuparna rx +rxnave +rxnsum ry rz Ryckaert @@ -3103,6 +3123,7 @@ Swinburne Swol Swope Sx +sx sy Sy symplectic @@ -3112,6 +3133,7 @@ sysdim Syst systemd Sz +sz Tabbernor tabinner Tadmor @@ -3130,6 +3152,7 @@ Tanmoy Tartakovsky taskset taubi +taylor tb tchain Tchain @@ -3163,6 +3186,7 @@ th thb thei Theodorou +Theophile Theor thermalization thermalize @@ -3315,6 +3339,7 @@ Uleft uloop Ulomek ulsph +Ultrafast uMech umin Umin @@ -3350,6 +3375,7 @@ upenn upto Urbakh Urbana +Usabiaga usec uSemiParallel userguide @@ -3494,6 +3520,7 @@ Wikipedia Wildcard wildcard wildcards +Winkler Wirnsberger wirtes witin @@ -3505,6 +3532,7 @@ Worley Wriggers Wuppertal Wurtzite +Wysocki www wx Wx @@ -3595,6 +3623,7 @@ ZBL Zc zcm Zeeman +zeeman Zemer Zepeda zflag diff --git a/examples/ASPHERE/box/in.box b/examples/ASPHERE/box/in.box index 2ec09faf56..6018458231 100644 --- a/examples/ASPHERE/box/in.box +++ b/examples/ASPHERE/box/in.box @@ -67,7 +67,7 @@ delete_atoms overlap 1.0 small big reset_timestep 0 -neighbor 0.3 bin +neighbor 0.3 multi neigh_modify delay 0 every 1 check yes comm_modify mode multi group big vel yes diff --git a/examples/ASPHERE/box/in.box.mp b/examples/ASPHERE/box/in.box.mp index 2953552595..ce1591c7dc 100644 --- a/examples/ASPHERE/box/in.box.mp +++ b/examples/ASPHERE/box/in.box.mp @@ -67,7 +67,7 @@ delete_atoms overlap 1.0 small big reset_timestep 0 -neighbor 0.3 bin +neighbor 0.3 multi neigh_modify delay 0 every 1 check yes comm_modify mode multi group big vel yes diff --git a/examples/ASPHERE/dimer/in.dimer b/examples/ASPHERE/dimer/in.dimer index 9e514d83f1..224db9937d 100644 --- a/examples/ASPHERE/dimer/in.dimer +++ b/examples/ASPHERE/dimer/in.dimer @@ -61,7 +61,7 @@ delete_atoms overlap 0.5 small big reset_timestep 0 -neighbor 0.3 bin +neighbor 0.3 multi neigh_modify delay 0 every 1 check yes comm_modify mode multi group big vel yes diff --git a/examples/ASPHERE/dimer/in.dimer.mp b/examples/ASPHERE/dimer/in.dimer.mp index 2970df022b..a637acaf87 100644 --- a/examples/ASPHERE/dimer/in.dimer.mp +++ b/examples/ASPHERE/dimer/in.dimer.mp @@ -61,7 +61,7 @@ delete_atoms overlap 0.5 small big reset_timestep 0 -neighbor 0.3 bin +neighbor 0.3 multi neigh_modify delay 0 every 1 check yes comm_modify mode multi group big vel yes diff --git a/examples/ASPHERE/ellipsoid/in.ellipsoid b/examples/ASPHERE/ellipsoid/in.ellipsoid index 241f010771..5e9a20f6e0 100644 --- a/examples/ASPHERE/ellipsoid/in.ellipsoid +++ b/examples/ASPHERE/ellipsoid/in.ellipsoid @@ -72,7 +72,7 @@ delete_atoms overlap 1.6 small big reset_timestep 0 -neighbor 0.3 bin +neighbor 0.3 multi neigh_modify delay 0 every 1 check yes comm_modify mode multi group big vel yes diff --git a/examples/ASPHERE/ellipsoid/in.ellipsoid.mp b/examples/ASPHERE/ellipsoid/in.ellipsoid.mp index 52c30db0a9..7ce01ad4e1 100644 --- a/examples/ASPHERE/ellipsoid/in.ellipsoid.mp +++ b/examples/ASPHERE/ellipsoid/in.ellipsoid.mp @@ -72,7 +72,7 @@ delete_atoms overlap 1.6 small big reset_timestep 0 -neighbor 0.3 bin +neighbor 0.3 multi neigh_modify delay 0 every 1 check yes comm_modify mode multi group big vel yes diff --git a/examples/ASPHERE/line/in.line.srd b/examples/ASPHERE/line/in.line.srd index 6ec1998d31..a32bde0b76 100644 --- a/examples/ASPHERE/line/in.line.srd +++ b/examples/ASPHERE/line/in.line.srd @@ -34,7 +34,7 @@ reset_timestep 0 velocity small create 1.44 87287 loop geom -neighbor 0.3 bin +neighbor 0.3 multi neigh_modify delay 0 every 1 check yes neigh_modify exclude molecule/intra big include big diff --git a/examples/ASPHERE/poly/in.poly b/examples/ASPHERE/poly/in.poly index 3496a774bb..68bd2a6bbc 100644 --- a/examples/ASPHERE/poly/in.poly +++ b/examples/ASPHERE/poly/in.poly @@ -70,7 +70,7 @@ delete_atoms overlap 1.0 small big reset_timestep 0 -neighbor 0.3 bin +neighbor 0.3 multi neigh_modify delay 0 every 1 check yes comm_modify mode multi group big vel yes diff --git a/examples/ASPHERE/poly/in.poly.mp b/examples/ASPHERE/poly/in.poly.mp index 1c6a1faee3..8fa5d24a55 100644 --- a/examples/ASPHERE/poly/in.poly.mp +++ b/examples/ASPHERE/poly/in.poly.mp @@ -70,7 +70,7 @@ delete_atoms overlap 1.0 small big reset_timestep 0 -neighbor 0.3 bin +neighbor 0.3 multi neigh_modify delay 0 every 1 check yes comm_modify mode multi group big vel yes diff --git a/examples/ASPHERE/star/in.star b/examples/ASPHERE/star/in.star index 6711e7a0a7..a098810199 100644 --- a/examples/ASPHERE/star/in.star +++ b/examples/ASPHERE/star/in.star @@ -67,7 +67,7 @@ delete_atoms overlap 0.5 small big reset_timestep 0 -neighbor 0.3 bin +neighbor 0.3 multi neigh_modify delay 0 every 1 check yes comm_modify mode multi group big vel yes diff --git a/examples/ASPHERE/star/in.star.mp b/examples/ASPHERE/star/in.star.mp index de5f23dfb6..53cb924e42 100644 --- a/examples/ASPHERE/star/in.star.mp +++ b/examples/ASPHERE/star/in.star.mp @@ -67,7 +67,7 @@ delete_atoms overlap 0.5 small big reset_timestep 0 -neighbor 0.3 bin +neighbor 0.3 multi neigh_modify delay 0 every 1 check yes comm_modify mode multi group big vel yes diff --git a/examples/ASPHERE/tri/in.tri.srd b/examples/ASPHERE/tri/in.tri.srd index e7ef6ef59a..9bfc9d7b65 100644 --- a/examples/ASPHERE/tri/in.tri.srd +++ b/examples/ASPHERE/tri/in.tri.srd @@ -33,7 +33,7 @@ reset_timestep 0 velocity small create 1.44 87287 loop geom -neighbor 0.3 bin +neighbor 0.3 multi neigh_modify delay 0 every 1 check yes neigh_modify exclude molecule/intra big include big diff --git a/examples/USER/brownian/2d_velocity/in2d.velocity b/examples/USER/brownian/2d_velocity/in2d.velocity new file mode 100644 index 0000000000..fcfae0554b --- /dev/null +++ b/examples/USER/brownian/2d_velocity/in2d.velocity @@ -0,0 +1,53 @@ +##### 2d overdamped brownian dynamics with self-propulsion force in direction of velocity. ##### + +variable gamma_t equal 1.0 +variable temp equal 1.0 +variable seed equal 1974019 +variable fp equal 4.0 +variable params string ${gamma_t}_${temp}_${fp} + +units lj +dimension 2 +newton off + +lattice sq 0.4 +region box block -16 16 -16 16 -0.2 0.2 +create_box 1 box +create_atoms 1 box +mass * 1.0 +velocity all create 1.0 1 loop geom + +neighbor 1.0 bin +neigh_modify every 1 delay 1 check yes + +pair_style none + +fix step all brownian ${temp} ${seed} gamma_t ${gamma_t} +fix vel all propel/self velocity ${fp} +fix 2 all enforce2d +fix_modify vel virial yes + +compute press all pressure NULL virial + +thermo_style custom step temp epair c_press + +#equilibration +timestep 0.0000000001 +thermo 500 +run 5000 +reset_timestep 0 + +#initialisation for the main run + +# MSD +compute msd all msd + +thermo_style custom step ke pe c_msd[*] c_press + +timestep 0.00001 +thermo 1000 + +# main run +run 12000 + + diff --git a/examples/USER/brownian/2d_velocity/log.11May2021.in2d_velocity.g++.1 b/examples/USER/brownian/2d_velocity/log.11May2021.in2d_velocity.g++.1 new file mode 100644 index 0000000000..a5a58ba9f0 --- /dev/null +++ b/examples/USER/brownian/2d_velocity/log.11May2021.in2d_velocity.g++.1 @@ -0,0 +1,151 @@ +LAMMPS (8 Apr 2021) + using 1 OpenMP thread(s) per MPI task +##### 2d overdamped brownian dynamics with self-propulsion force in direction of velocity. ##### + +variable gamma_t equal 1.0 +variable temp equal 1.0 +variable seed equal 1974019 +variable fp equal 4.0 +variable params string ${gamma_t}_${temp}_${fp} +variable params string 1_${temp}_${fp} +variable params string 1_1_${fp} +variable params string 1_1_4 + +units lj +dimension 2 +newton off + +lattice sq 0.4 +Lattice spacing in x,y,z = 1.5811388 1.5811388 1.5811388 +region box block -16 16 -16 16 -0.2 0.2 +create_box 1 box +Created orthogonal box = (-25.298221 -25.298221 -0.31622777) to (25.298221 25.298221 0.31622777) + 1 by 1 by 1 MPI processor grid +create_atoms 1 box +Created 1024 atoms + create_atoms CPU = 0.001 seconds +mass * 1.0 +velocity all create 1.0 1 loop geom + +neighbor 1.0 bin +neigh_modify every 1 delay 1 check yes + +pair_style none + +fix step all brownian ${temp} ${seed} gamma_t ${gamma_t} +fix step all brownian 1 ${seed} gamma_t ${gamma_t} +fix step all brownian 1 1974019 gamma_t ${gamma_t} +fix step all brownian 1 1974019 gamma_t 1 +fix vel all propel/self velocity ${fp} +fix vel all propel/self velocity 4 +fix 2 all enforce2d +fix_modify vel virial yes + +compute press all pressure NULL virial + +thermo_style custom step temp epair c_press + +#equilibration +timestep 0.0000000001 +thermo 500 +run 5000 +WARNING: No pairwise cutoff or binsize set. Atom sorting therefore disabled. (src/atom.cpp:2141) +WARNING: Communication cutoff is 0.0. No ghost atoms will be generated. Atoms may get lost. (src/comm_brick.cpp:167) +Per MPI rank memory allocation (min/avg/max) = 2.289 | 2.289 | 2.289 Mbytes +Step Temp E_pair c_press + 0 1 0 -0.18336111 + 500 2.0519273e+10 0 -0.048238222 + 1000 1.9821717e+10 0 -0.4711053 + 1500 1.9697609e+10 0 -0.13539588 + 2000 2.0209443e+10 0 0.0094958039 + 2500 1.9591299e+10 0 0.40117118 + 3000 2.089566e+10 0 -0.036548251 + 3500 1.978692e+10 0 0.28282578 + 4000 2.0657848e+10 0 0.17618064 + 4500 2.0837353e+10 0 -0.080724651 + 5000 2.0348316e+10 0 -0.17471195 +Loop time of 0.575164 on 1 procs for 5000 steps with 1024 atoms + +Performance: 0.075 tau/day, 8693.168 timesteps/s +99.9% CPU use with 1 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 0 | 0 | 0 | 0.0 | 0.00 +Neigh | 0 | 0 | 0 | 0.0 | 0.00 +Comm | 0.0036819 | 0.0036819 | 0.0036819 | 0.0 | 0.64 +Output | 0.00027752 | 0.00027752 | 0.00027752 | 0.0 | 0.05 +Modify | 0.51999 | 0.51999 | 0.51999 | 0.0 | 90.41 +Other | | 0.05121 | | | 8.90 + +Nlocal: 1024.00 ave 1024 max 1024 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Nghost: 65.0000 ave 65 max 65 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Neighs: 0.00000 ave 0 max 0 min +Histogram: 1 0 0 0 0 0 0 0 0 0 + +Total # of neighbors = 0 +Ave neighs/atom = 0.0000000 +Neighbor list builds = 0 +Dangerous builds = 0 +reset_timestep 0 + +#initialisation for the main run + +# MSD +compute msd all msd + +thermo_style custom step ke pe c_msd[*] c_press + +timestep 0.00001 +thermo 1000 + +# main run +run 12000 +WARNING: Communication cutoff is 0.0. No ghost atoms will be generated. Atoms may get lost. (src/comm_brick.cpp:167) +Per MPI rank memory allocation (min/avg/max) = 2.664 | 2.664 | 2.664 Mbytes +Step KinEng PotEng c_msd[1] c_msd[2] c_msd[3] c_msd[4] c_press + 0 2.0328444e+10 0 0 0 0 0 -0.17471195 + 1000 197017.59 0 0.018147562 0.019839233 0 0.037986796 -0.71897807 + 2000 197030.23 0 0.03909867 0.041721342 0 0.080820011 -0.30051929 + 3000 201997.2 0 0.065694399 0.06235257 0 0.12804697 -0.85167039 + 4000 199927.76 0 0.085698715 0.080328815 0 0.16602753 0.18493117 + 5000 198665.7 0 0.10896054 0.097021266 0 0.2059818 -0.090735406 + 6000 199277.78 0 0.13081111 0.11724814 0 0.24805925 -0.18189034 + 7000 199850.54 0 0.14721838 0.13806858 0 0.28528696 0.11334674 + 8000 191577.11 0 0.16582149 0.15935853 0 0.32518002 -0.73284569 + 9000 197331.29 0 0.17995704 0.18652927 0 0.3664863 -0.015558407 + 10000 197048.17 0 0.2034106 0.20329856 0 0.40670916 0.36985211 + 11000 200105.54 0 0.21809835 0.21966463 0 0.43776298 0.36437 + 12000 203180.39 0 0.23810386 0.23666184 0 0.47476569 -0.072006034 +Loop time of 1.37465 on 1 procs for 12000 steps with 1024 atoms + +Performance: 7542.303 tau/day, 8729.517 timesteps/s +99.8% CPU use with 1 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 0 | 0 | 0 | 0.0 | 0.00 +Neigh | 0.00012231 | 0.00012231 | 0.00012231 | 0.0 | 0.01 +Comm | 0.0024607 | 0.0024607 | 0.0024607 | 0.0 | 0.18 +Output | 0.00068665 | 0.00068665 | 0.00068665 | 0.0 | 0.05 +Modify | 1.2479 | 1.2479 | 1.2479 | 0.0 | 90.78 +Other | | 0.1235 | | | 8.98 + +Nlocal: 1024.00 ave 1024 max 1024 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Nghost: 0.00000 ave 0 max 0 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Neighs: 0.00000 ave 0 max 0 min +Histogram: 1 0 0 0 0 0 0 0 0 0 + +Total # of neighbors = 0 +Ave neighs/atom = 0.0000000 +Neighbor list builds = 15 +Dangerous builds = 0 + + +Total wall time: 0:00:01 diff --git a/examples/USER/brownian/2d_velocity/log.11May2021.in2d_velocity.g++.4 b/examples/USER/brownian/2d_velocity/log.11May2021.in2d_velocity.g++.4 new file mode 100644 index 0000000000..df9ff24070 --- /dev/null +++ b/examples/USER/brownian/2d_velocity/log.11May2021.in2d_velocity.g++.4 @@ -0,0 +1,151 @@ +LAMMPS (8 Apr 2021) + using 1 OpenMP thread(s) per MPI task +##### 2d overdamped brownian dynamics with self-propulsion force in direction of velocity. ##### + +variable gamma_t equal 1.0 +variable temp equal 1.0 +variable seed equal 1974019 +variable fp equal 4.0 +variable params string ${gamma_t}_${temp}_${fp} +variable params string 1_${temp}_${fp} +variable params string 1_1_${fp} +variable params string 1_1_4 + +units lj +dimension 2 +newton off + +lattice sq 0.4 +Lattice spacing in x,y,z = 1.5811388 1.5811388 1.5811388 +region box block -16 16 -16 16 -0.2 0.2 +create_box 1 box +Created orthogonal box = (-25.298221 -25.298221 -0.31622777) to (25.298221 25.298221 0.31622777) + 2 by 2 by 1 MPI processor grid +create_atoms 1 box +Created 1024 atoms + create_atoms CPU = 0.001 seconds +mass * 1.0 +velocity all create 1.0 1 loop geom + +neighbor 1.0 bin +neigh_modify every 1 delay 1 check yes + +pair_style none + +fix step all brownian ${temp} ${seed} gamma_t ${gamma_t} +fix step all brownian 1 ${seed} gamma_t ${gamma_t} +fix step all brownian 1 1974019 gamma_t ${gamma_t} +fix step all brownian 1 1974019 gamma_t 1 +fix vel all propel/self velocity ${fp} +fix vel all propel/self velocity 4 +fix 2 all enforce2d +fix_modify vel virial yes + +compute press all pressure NULL virial + +thermo_style custom step temp epair c_press + +#equilibration +timestep 0.0000000001 +thermo 500 +run 5000 +WARNING: No pairwise cutoff or binsize set. Atom sorting therefore disabled. (src/atom.cpp:2141) +WARNING: Communication cutoff is 0.0. No ghost atoms will be generated. Atoms may get lost. (src/comm_brick.cpp:167) +Per MPI rank memory allocation (min/avg/max) = 2.289 | 2.289 | 2.289 Mbytes +Step Temp E_pair c_press + 0 1 0 -0.18336111 + 500 1.9862591e+10 0 -0.32013566 + 1000 2.0093184e+10 0 -0.36609742 + 1500 1.9562283e+10 0 -0.53349351 + 2000 1.9903977e+10 0 0.63783249 + 2500 2.0260128e+10 0 0.30046413 + 3000 1.9948065e+10 0 -0.63093105 + 3500 1.9507486e+10 0 0.48762848 + 4000 2.0049087e+10 0 0.40289309 + 4500 1.9975813e+10 0 0.57649363 + 5000 2.0129291e+10 0 -0.41288352 +Loop time of 0.238949 on 4 procs for 5000 steps with 1024 atoms + +Performance: 0.181 tau/day, 20924.952 timesteps/s +92.6% CPU use with 4 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 0 | 0 | 0 | 0.0 | 0.00 +Neigh | 0 | 0 | 0 | 0.0 | 0.00 +Comm | 0.0080078 | 0.024718 | 0.031782 | 6.2 | 10.34 +Output | 0.0001812 | 0.00029999 | 0.00063467 | 0.0 | 0.13 +Modify | 0.13401 | 0.14401 | 0.15438 | 2.4 | 60.27 +Other | | 0.06992 | | | 29.26 + +Nlocal: 256.000 ave 256 max 256 min +Histogram: 4 0 0 0 0 0 0 0 0 0 +Nghost: 33.0000 ave 33 max 33 min +Histogram: 4 0 0 0 0 0 0 0 0 0 +Neighs: 0.00000 ave 0 max 0 min +Histogram: 4 0 0 0 0 0 0 0 0 0 + +Total # of neighbors = 0 +Ave neighs/atom = 0.0000000 +Neighbor list builds = 0 +Dangerous builds = 0 +reset_timestep 0 + +#initialisation for the main run + +# MSD +compute msd all msd + +thermo_style custom step ke pe c_msd[*] c_press + +timestep 0.00001 +thermo 1000 + +# main run +run 12000 +WARNING: Communication cutoff is 0.0. No ghost atoms will be generated. Atoms may get lost. (src/comm_brick.cpp:167) +Per MPI rank memory allocation (min/avg/max) = 2.664 | 2.664 | 2.664 Mbytes +Step KinEng PotEng c_msd[1] c_msd[2] c_msd[3] c_msd[4] c_press + 0 2.0109634e+10 0 0 0 0 0 -0.41288352 + 1000 195711.46 0 0.020076462 0.020523099 0 0.040599561 -0.32125126 + 2000 203263.85 0 0.039242992 0.039661282 0 0.078904274 0.11008705 + 3000 197417.54 0 0.064938128 0.057716419 0 0.12265455 0.16967601 + 4000 200505.97 0 0.086511225 0.074975267 0 0.16148649 0.31338473 + 5000 199373.77 0 0.10583263 0.098175658 0 0.20400829 0.34205791 + 6000 192881.14 0 0.12152088 0.11706037 0 0.23858125 -0.27870467 + 7000 203045.3 0 0.1383248 0.13629503 0 0.27461983 -0.046936646 + 8000 198544.08 0 0.16064738 0.1582206 0 0.31886798 -0.18803452 + 9000 205450.74 0 0.17926529 0.1829047 0 0.36216999 0.47191228 + 10000 200371.73 0 0.20084273 0.20365189 0 0.40449463 0.093098262 + 11000 202911.93 0 0.21569236 0.22221715 0 0.43790952 -0.38430031 + 12000 192590.04 0 0.24041439 0.24114487 0 0.48155926 -0.1677052 +Loop time of 0.443026 on 4 procs for 12000 steps with 1024 atoms + +Performance: 23402.683 tau/day, 27086.439 timesteps/s +97.7% CPU use with 4 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 0 | 0 | 0 | 0.0 | 0.00 +Neigh | 3.2663e-05 | 3.3855e-05 | 3.4809e-05 | 0.0 | 0.01 +Comm | 0.0030291 | 0.0030628 | 0.0030825 | 0.0 | 0.69 +Output | 0.00027895 | 0.00051624 | 0.001184 | 0.0 | 0.12 +Modify | 0.31607 | 0.33372 | 0.37391 | 4.0 | 75.33 +Other | | 0.1057 | | | 23.86 + +Nlocal: 256.000 ave 259 max 253 min +Histogram: 1 0 0 1 0 0 1 0 0 1 +Nghost: 0.00000 ave 0 max 0 min +Histogram: 4 0 0 0 0 0 0 0 0 0 +Neighs: 0.00000 ave 0 max 0 min +Histogram: 4 0 0 0 0 0 0 0 0 0 + +Total # of neighbors = 0 +Ave neighs/atom = 0.0000000 +Neighbor list builds = 15 +Dangerous builds = 0 + + +Total wall time: 0:00:00 diff --git a/examples/USER/brownian/asphere/in2d.ellipsoid b/examples/USER/brownian/asphere/in2d.ellipsoid new file mode 100644 index 0000000000..4b1a88bba7 --- /dev/null +++ b/examples/USER/brownian/asphere/in2d.ellipsoid @@ -0,0 +1,54 @@ +##### overdamped dynamics of non-interacting ellipsoids in 2D ##### + +variable rng string gaussian +variable seed string 198098 +variable temp string 1.0 +variable gamma_r_1 string inf +variable gamma_r_2 string inf +variable gamma_r_3 string 0.1 +variable gamma_t_1 string 5.0 +variable gamma_t_2 string 7.0 +variable gamma_t_3 string inf +variable params string ${rng}_${temp}_${gamma_r_1}_${gamma_r_2}_${gamma_r_3}_${gamma_t_1}_${gamma_t_2}_${gamma_t_3} + +units lj +atom_style hybrid dipole ellipsoid +dimension 2 +newton off + +lattice sq 0.4 +region box block -30 30 -30 30 -0.2 0.2 +create_box 1 box +create_atoms 1 box +mass * 1.0 +set type * dipole/random ${seed} 1.0 +set type * shape 3.0 1.0 1.0 +set type * quat/random ${seed} +velocity all create 1.0 1 loop geom + +neighbor 1.0 bin +neigh_modify every 1 delay 1 check yes + +pair_style none + +fix 1 all brownian/asphere ${temp} ${seed} rng ${rng} & + gamma_r_eigen ${gamma_r_1} ${gamma_r_2} ${gamma_r_3} & + gamma_t_eigen ${gamma_t_1} ${gamma_t_2} ${gamma_t_3} & + dipole 1.0 0.0 0.0 + +#initialisation for the main run + +# MSD +compute msd all msd + +thermo_style custom step ke pe c_msd[*] + +#dump 1 all custom 1000 dump_${params}_2d.lammpstrj id type & +# x y z xu yu zu mux muy muz fx fy fz +#dump_modify 1 first yes sort id + +timestep 0.00001 +thermo 100 + +# main run +run 3000 diff --git a/examples/USER/brownian/asphere/in3d.ellipsoid b/examples/USER/brownian/asphere/in3d.ellipsoid new file mode 100644 index 0000000000..34f2684ee7 --- /dev/null +++ b/examples/USER/brownian/asphere/in3d.ellipsoid @@ -0,0 +1,54 @@ +##### overdamped dynamics of non-interacting ellipsoids in 3D ##### + +variable rng string uniform +variable seed string 198098 +variable temp string 1.0 +variable gamma_r_1 string 2.0 +variable gamma_r_2 string 0.25 +variable gamma_r_3 string 0.1 +variable gamma_t_1 string 5.0 +variable gamma_t_2 string 7.0 +variable gamma_t_3 string 9.0 +variable params string ${rng}_${temp}_${gamma_r_1}_${gamma_r_2}_${gamma_r_3}_${gamma_t_1}_${gamma_t_2}_${gamma_t_3} + +units lj +atom_style hybrid dipole ellipsoid +dimension 3 +newton off + +lattice sc 0.4 +region box block -8 8 -8 8 -8 8 +create_box 1 box +create_atoms 1 box +mass * 1.0 +set type * dipole/random ${seed} 1.0 +set type * shape 3.0 1.0 1.0 +set type * quat/random ${seed} +velocity all create 1.0 1 loop geom + +neighbor 1.0 bin +neigh_modify every 1 delay 1 check yes + +pair_style none + +fix 1 all brownian/asphere ${temp} ${seed} rng ${rng} & + gamma_r_eigen ${gamma_r_1} ${gamma_r_2} ${gamma_r_3} & + gamma_t_eigen ${gamma_t_1} ${gamma_t_2} ${gamma_t_3} & + dipole 1.0 0.0 0.0 + +#initialisation for the main run + +# MSD +compute msd all msd + +thermo_style custom step ke pe c_msd[*] + +#dump 1 all custom 1000 dump_${params}_3d.lammpstrj id type & +# x y z xu yu zu mux muy muz fx fy fz +#dump_modify 1 first yes sort id + +timestep 0.00001 +thermo 100 + +# main run +run 3000 diff --git a/examples/USER/brownian/asphere/log.11May2021.in2d.ellipsoid.g++.1 b/examples/USER/brownian/asphere/log.11May2021.in2d.ellipsoid.g++.1 new file mode 100644 index 0000000000..619ab937c9 --- /dev/null +++ b/examples/USER/brownian/asphere/log.11May2021.in2d.ellipsoid.g++.1 @@ -0,0 +1,145 @@ +LAMMPS (8 Apr 2021) + using 1 OpenMP thread(s) per MPI task +##### overdamped dynamics of non-interacting ellipsoids in 2D ##### + +variable rng string gaussian +variable seed string 198098 +variable temp string 1.0 +variable gamma_r_1 string inf +variable gamma_r_2 string inf +variable gamma_r_3 string 0.1 +variable gamma_t_1 string 5.0 +variable gamma_t_2 string 7.0 +variable gamma_t_3 string inf +variable params string ${rng}_${temp}_${gamma_r_1}_${gamma_r_2}_${gamma_r_3}_${gamma_t_1}_${gamma_t_2}_${gamma_t_3} +variable params string gaussian_${temp}_${gamma_r_1}_${gamma_r_2}_${gamma_r_3}_${gamma_t_1}_${gamma_t_2}_${gamma_t_3} +variable params string gaussian_1.0_${gamma_r_1}_${gamma_r_2}_${gamma_r_3}_${gamma_t_1}_${gamma_t_2}_${gamma_t_3} +variable params string gaussian_1.0_inf_${gamma_r_2}_${gamma_r_3}_${gamma_t_1}_${gamma_t_2}_${gamma_t_3} +variable params string gaussian_1.0_inf_inf_${gamma_r_3}_${gamma_t_1}_${gamma_t_2}_${gamma_t_3} +variable params string gaussian_1.0_inf_inf_0.1_${gamma_t_1}_${gamma_t_2}_${gamma_t_3} +variable params string gaussian_1.0_inf_inf_0.1_5.0_${gamma_t_2}_${gamma_t_3} +variable params string gaussian_1.0_inf_inf_0.1_5.0_7.0_${gamma_t_3} +variable params string gaussian_1.0_inf_inf_0.1_5.0_7.0_inf + +units lj +atom_style hybrid dipole ellipsoid +WARNING: Atom style hybrid defines both, per-type and per-atom masses; both must be set, but only per-atom masses will be used (src/atom_vec_hybrid.cpp:156) +dimension 2 +newton off + +lattice sq 0.4 +Lattice spacing in x,y,z = 1.5811388 1.5811388 1.5811388 +region box block -30 30 -30 30 -0.2 0.2 +create_box 1 box +Created orthogonal box = (-47.434165 -47.434165 -0.31622777) to (47.434165 47.434165 0.31622777) + 1 by 1 by 1 MPI processor grid +create_atoms 1 box +Created 3600 atoms + create_atoms CPU = 0.005 seconds +mass * 1.0 +set type * dipole/random ${seed} 1.0 +set type * dipole/random 198098 1.0 +Setting atom values ... + 3600 settings made for dipole/random +set type * shape 3.0 1.0 1.0 +Setting atom values ... + 3600 settings made for shape +set type * quat/random ${seed} +set type * quat/random 198098 +Setting atom values ... + 3600 settings made for quat/random +velocity all create 1.0 1 loop geom + +neighbor 1.0 bin +neigh_modify every 1 delay 1 check yes + +pair_style none + +fix 1 all brownian/asphere ${temp} ${seed} rng ${rng} gamma_r_eigen ${gamma_r_1} ${gamma_r_2} ${gamma_r_3} gamma_t_eigen ${gamma_t_1} ${gamma_t_2} ${gamma_t_3} dipole 1.0 0.0 0.0 +fix 1 all brownian/asphere 1.0 ${seed} rng ${rng} gamma_r_eigen ${gamma_r_1} ${gamma_r_2} ${gamma_r_3} gamma_t_eigen ${gamma_t_1} ${gamma_t_2} ${gamma_t_3} dipole 1.0 0.0 0.0 +fix 1 all brownian/asphere 1.0 198098 rng ${rng} gamma_r_eigen ${gamma_r_1} ${gamma_r_2} ${gamma_r_3} gamma_t_eigen ${gamma_t_1} ${gamma_t_2} ${gamma_t_3} dipole 1.0 0.0 0.0 +fix 1 all brownian/asphere 1.0 198098 rng gaussian gamma_r_eigen ${gamma_r_1} ${gamma_r_2} ${gamma_r_3} gamma_t_eigen ${gamma_t_1} ${gamma_t_2} ${gamma_t_3} dipole 1.0 0.0 0.0 +fix 1 all brownian/asphere 1.0 198098 rng gaussian gamma_r_eigen inf ${gamma_r_2} ${gamma_r_3} gamma_t_eigen ${gamma_t_1} ${gamma_t_2} ${gamma_t_3} dipole 1.0 0.0 0.0 +fix 1 all brownian/asphere 1.0 198098 rng gaussian gamma_r_eigen inf inf ${gamma_r_3} gamma_t_eigen ${gamma_t_1} ${gamma_t_2} ${gamma_t_3} dipole 1.0 0.0 0.0 +fix 1 all brownian/asphere 1.0 198098 rng gaussian gamma_r_eigen inf inf 0.1 gamma_t_eigen ${gamma_t_1} ${gamma_t_2} ${gamma_t_3} dipole 1.0 0.0 0.0 +fix 1 all brownian/asphere 1.0 198098 rng gaussian gamma_r_eigen inf inf 0.1 gamma_t_eigen 5.0 ${gamma_t_2} ${gamma_t_3} dipole 1.0 0.0 0.0 +fix 1 all brownian/asphere 1.0 198098 rng gaussian gamma_r_eigen inf inf 0.1 gamma_t_eigen 5.0 7.0 ${gamma_t_3} dipole 1.0 0.0 0.0 +fix 1 all brownian/asphere 1.0 198098 rng gaussian gamma_r_eigen inf inf 0.1 gamma_t_eigen 5.0 7.0 inf dipole 1.0 0.0 0.0 + +#initialisation for the main run + +# MSD +compute msd all msd + +thermo_style custom step ke pe c_msd[*] + +#dump 1 all custom 1000 dump_${params}_2d.lammpstrj id type # x y z xu yu zu mux muy muz fx fy fz +#dump_modify 1 first yes sort id + +timestep 0.00001 +thermo 100 + +# main run +run 3000 +WARNING: No pairwise cutoff or binsize set. Atom sorting therefore disabled. (src/atom.cpp:2141) +WARNING: Communication cutoff is 0.0. No ghost atoms will be generated. Atoms may get lost. (src/comm_brick.cpp:167) +Per MPI rank memory allocation (min/avg/max) = 5.114 | 5.114 | 5.114 Mbytes +Step KinEng PotEng c_msd[1] c_msd[2] c_msd[3] c_msd[4] + 0 0.99972222 0 0 0 0 0 + 100 34376.187 0 0.00034728749 0.00034318997 0 0.00069047747 + 200 36135.708 0 0.00067452936 0.0006818928 0 0.0013564222 + 300 34444.929 0 0.0010189319 0.00099849203 0 0.002017424 + 400 35668.312 0 0.0013648699 0.0013311884 0 0.0026960583 + 500 35388.615 0 0.0017330203 0.0016077184 0 0.0033407387 + 600 35180.065 0 0.002052223 0.0019394635 0 0.0039916865 + 700 34035.38 0 0.0024329956 0.0022664905 0 0.0046994861 + 800 34581.664 0 0.002783885 0.0025794872 0 0.0053633723 + 900 34579.945 0 0.003163442 0.0029351952 0 0.0060986372 + 1000 34158.066 0 0.0035589034 0.0032627605 0 0.0068216639 + 1100 33453.827 0 0.0038861895 0.003565372 0 0.0074515615 + 1200 33608.06 0 0.0041325698 0.0038943268 0 0.0080268966 + 1300 34381.633 0 0.004405682 0.0043294156 0 0.0087350976 + 1400 32925.746 0 0.0047383547 0.0046803517 0 0.0094187065 + 1500 34809.764 0 0.0051149571 0.0049309746 0 0.010045932 + 1600 33580.096 0 0.0054893472 0.0052465377 0 0.010735885 + 1700 34596.275 0 0.00581894 0.0056500316 0 0.011468972 + 1800 33926.736 0 0.0062129617 0.0059796125 0 0.012192574 + 1900 35577.131 0 0.0065668637 0.0062530163 0 0.01281988 + 2000 34224.967 0 0.0070005917 0.006598912 0 0.013599504 + 2100 33991.406 0 0.0073134826 0.0069119252 0 0.014225408 + 2200 34647.054 0 0.007659301 0.0073434715 0 0.015002772 + 2300 33956.835 0 0.007965191 0.0076318537 0 0.015597045 + 2400 35272.549 0 0.0082467116 0.007929202 0 0.016175914 + 2500 33901.494 0 0.0086251299 0.0082790757 0 0.016904206 + 2600 34138.227 0 0.0089419364 0.0086639744 0 0.017605911 + 2700 33691.013 0 0.0093083376 0.0090219118 0 0.018330249 + 2800 34716.817 0 0.0095840095 0.0094118945 0 0.018995904 + 2900 34473.982 0 0.0099773501 0.0098167668 0 0.019794117 + 3000 33406.776 0 0.010391969 0.010098625 0 0.020490594 +Loop time of 3.67112 on 1 procs for 3000 steps with 3600 atoms + +Performance: 706.051 tau/day, 817.189 timesteps/s +99.9% CPU use with 1 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 0 | 0 | 0 | 0.0 | 0.00 +Neigh | 0 | 0 | 0 | 0.0 | 0.00 +Comm | 0.037973 | 0.037973 | 0.037973 | 0.0 | 1.03 +Output | 0.0040674 | 0.0040674 | 0.0040674 | 0.0 | 0.11 +Modify | 3.515 | 3.515 | 3.515 | 0.0 | 95.75 +Other | | 0.1141 | | | 3.11 + +Nlocal: 3600.00 ave 3600 max 3600 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Nghost: 121.000 ave 121 max 121 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Neighs: 0.00000 ave 0 max 0 min +Histogram: 1 0 0 0 0 0 0 0 0 0 + +Total # of neighbors = 0 +Ave neighs/atom = 0.0000000 +Neighbor list builds = 0 +Dangerous builds = 0 +Total wall time: 0:00:03 diff --git a/examples/USER/brownian/asphere/log.11May2021.in2d.ellipsoid.g++.4 b/examples/USER/brownian/asphere/log.11May2021.in2d.ellipsoid.g++.4 new file mode 100644 index 0000000000..b3da385dda --- /dev/null +++ b/examples/USER/brownian/asphere/log.11May2021.in2d.ellipsoid.g++.4 @@ -0,0 +1,145 @@ +LAMMPS (8 Apr 2021) + using 1 OpenMP thread(s) per MPI task +##### overdamped dynamics of non-interacting ellipsoids in 2D ##### + +variable rng string gaussian +variable seed string 198098 +variable temp string 1.0 +variable gamma_r_1 string inf +variable gamma_r_2 string inf +variable gamma_r_3 string 0.1 +variable gamma_t_1 string 5.0 +variable gamma_t_2 string 7.0 +variable gamma_t_3 string inf +variable params string ${rng}_${temp}_${gamma_r_1}_${gamma_r_2}_${gamma_r_3}_${gamma_t_1}_${gamma_t_2}_${gamma_t_3} +variable params string gaussian_${temp}_${gamma_r_1}_${gamma_r_2}_${gamma_r_3}_${gamma_t_1}_${gamma_t_2}_${gamma_t_3} +variable params string gaussian_1.0_${gamma_r_1}_${gamma_r_2}_${gamma_r_3}_${gamma_t_1}_${gamma_t_2}_${gamma_t_3} +variable params string gaussian_1.0_inf_${gamma_r_2}_${gamma_r_3}_${gamma_t_1}_${gamma_t_2}_${gamma_t_3} +variable params string gaussian_1.0_inf_inf_${gamma_r_3}_${gamma_t_1}_${gamma_t_2}_${gamma_t_3} +variable params string gaussian_1.0_inf_inf_0.1_${gamma_t_1}_${gamma_t_2}_${gamma_t_3} +variable params string gaussian_1.0_inf_inf_0.1_5.0_${gamma_t_2}_${gamma_t_3} +variable params string gaussian_1.0_inf_inf_0.1_5.0_7.0_${gamma_t_3} +variable params string gaussian_1.0_inf_inf_0.1_5.0_7.0_inf + +units lj +atom_style hybrid dipole ellipsoid +WARNING: Atom style hybrid defines both, per-type and per-atom masses; both must be set, but only per-atom masses will be used (src/atom_vec_hybrid.cpp:156) +dimension 2 +newton off + +lattice sq 0.4 +Lattice spacing in x,y,z = 1.5811388 1.5811388 1.5811388 +region box block -30 30 -30 30 -0.2 0.2 +create_box 1 box +Created orthogonal box = (-47.434165 -47.434165 -0.31622777) to (47.434165 47.434165 0.31622777) + 2 by 2 by 1 MPI processor grid +create_atoms 1 box +Created 3600 atoms + create_atoms CPU = 0.007 seconds +mass * 1.0 +set type * dipole/random ${seed} 1.0 +set type * dipole/random 198098 1.0 +Setting atom values ... + 3600 settings made for dipole/random +set type * shape 3.0 1.0 1.0 +Setting atom values ... + 3600 settings made for shape +set type * quat/random ${seed} +set type * quat/random 198098 +Setting atom values ... + 3600 settings made for quat/random +velocity all create 1.0 1 loop geom + +neighbor 1.0 bin +neigh_modify every 1 delay 1 check yes + +pair_style none + +fix 1 all brownian/asphere ${temp} ${seed} rng ${rng} gamma_r_eigen ${gamma_r_1} ${gamma_r_2} ${gamma_r_3} gamma_t_eigen ${gamma_t_1} ${gamma_t_2} ${gamma_t_3} dipole 1.0 0.0 0.0 +fix 1 all brownian/asphere 1.0 ${seed} rng ${rng} gamma_r_eigen ${gamma_r_1} ${gamma_r_2} ${gamma_r_3} gamma_t_eigen ${gamma_t_1} ${gamma_t_2} ${gamma_t_3} dipole 1.0 0.0 0.0 +fix 1 all brownian/asphere 1.0 198098 rng ${rng} gamma_r_eigen ${gamma_r_1} ${gamma_r_2} ${gamma_r_3} gamma_t_eigen ${gamma_t_1} ${gamma_t_2} ${gamma_t_3} dipole 1.0 0.0 0.0 +fix 1 all brownian/asphere 1.0 198098 rng gaussian gamma_r_eigen ${gamma_r_1} ${gamma_r_2} ${gamma_r_3} gamma_t_eigen ${gamma_t_1} ${gamma_t_2} ${gamma_t_3} dipole 1.0 0.0 0.0 +fix 1 all brownian/asphere 1.0 198098 rng gaussian gamma_r_eigen inf ${gamma_r_2} ${gamma_r_3} gamma_t_eigen ${gamma_t_1} ${gamma_t_2} ${gamma_t_3} dipole 1.0 0.0 0.0 +fix 1 all brownian/asphere 1.0 198098 rng gaussian gamma_r_eigen inf inf ${gamma_r_3} gamma_t_eigen ${gamma_t_1} ${gamma_t_2} ${gamma_t_3} dipole 1.0 0.0 0.0 +fix 1 all brownian/asphere 1.0 198098 rng gaussian gamma_r_eigen inf inf 0.1 gamma_t_eigen ${gamma_t_1} ${gamma_t_2} ${gamma_t_3} dipole 1.0 0.0 0.0 +fix 1 all brownian/asphere 1.0 198098 rng gaussian gamma_r_eigen inf inf 0.1 gamma_t_eigen 5.0 ${gamma_t_2} ${gamma_t_3} dipole 1.0 0.0 0.0 +fix 1 all brownian/asphere 1.0 198098 rng gaussian gamma_r_eigen inf inf 0.1 gamma_t_eigen 5.0 7.0 ${gamma_t_3} dipole 1.0 0.0 0.0 +fix 1 all brownian/asphere 1.0 198098 rng gaussian gamma_r_eigen inf inf 0.1 gamma_t_eigen 5.0 7.0 inf dipole 1.0 0.0 0.0 + +#initialisation for the main run + +# MSD +compute msd all msd + +thermo_style custom step ke pe c_msd[*] + +#dump 1 all custom 1000 dump_${params}_2d.lammpstrj id type # x y z xu yu zu mux muy muz fx fy fz +#dump_modify 1 first yes sort id + +timestep 0.00001 +thermo 100 + +# main run +run 3000 +WARNING: No pairwise cutoff or binsize set. Atom sorting therefore disabled. (src/atom.cpp:2141) +WARNING: Communication cutoff is 0.0. No ghost atoms will be generated. Atoms may get lost. (src/comm_brick.cpp:167) +Per MPI rank memory allocation (min/avg/max) = 5.102 | 5.102 | 5.102 Mbytes +Step KinEng PotEng c_msd[1] c_msd[2] c_msd[3] c_msd[4] + 0 0.99972222 0 0 0 0 0 + 100 33874.438 0 0.0003458147 0.00033165629 0 0.00067747099 + 200 34893.188 0 0.00066290209 0.00068146332 0 0.0013443654 + 300 34494.226 0 0.0010064902 0.0010276646 0 0.0020341548 + 400 34537.887 0 0.0013457339 0.0014057042 0 0.0027514381 + 500 34458.46 0 0.0017006949 0.0017120083 0 0.0034127033 + 600 33229.977 0 0.0020841613 0.0020485346 0 0.0041326959 + 700 33288.631 0 0.0024270272 0.0023673304 0 0.0047943576 + 800 35317.512 0 0.0027924435 0.0026950912 0 0.0054875347 + 900 33094.299 0 0.0031503627 0.0030789319 0 0.0062292946 + 1000 35801.751 0 0.003489398 0.0034594626 0 0.0069488607 + 1100 33427.701 0 0.0038547506 0.0038375809 0 0.0076923316 + 1200 34675.07 0 0.0041824195 0.0042017298 0 0.0083841493 + 1300 33080.294 0 0.0045258945 0.0045816356 0 0.0091075301 + 1400 34927.288 0 0.0048252992 0.0049215701 0 0.0097468693 + 1500 34338.558 0 0.0051959155 0.0053020102 0 0.010497926 + 1600 34686.248 0 0.0055111463 0.0056220225 0 0.011133169 + 1700 34336.158 0 0.0059240394 0.0059060319 0 0.011830071 + 1800 34315.859 0 0.0063027944 0.0063004467 0 0.012603241 + 1900 35096.721 0 0.0066098525 0.00672222 0 0.013332073 + 2000 33544.18 0 0.0069401261 0.007074124 0 0.01401425 + 2100 33863.219 0 0.0072726502 0.0074175954 0 0.014690246 + 2200 34705.892 0 0.0075586722 0.0077552683 0 0.015313941 + 2300 34025.357 0 0.0079046728 0.0081760519 0 0.016080725 + 2400 34741.849 0 0.008252969 0.0085203087 0 0.016773278 + 2500 34406.959 0 0.0085370091 0.0088556377 0 0.017392647 + 2600 34062.63 0 0.0088134153 0.0092536326 0 0.018067048 + 2700 34677.666 0 0.0090592854 0.0096225881 0 0.018681874 + 2800 33464.216 0 0.0093984162 0.0099647695 0 0.019363186 + 2900 32920.721 0 0.0098222985 0.010366517 0 0.020188816 + 3000 34539.66 0 0.010133317 0.01068102 0 0.020814337 +Loop time of 1.12143 on 4 procs for 3000 steps with 3600 atoms + +Performance: 2311.341 tau/day, 2675.163 timesteps/s +96.1% CPU use with 4 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 0 | 0 | 0 | 0.0 | 0.00 +Neigh | 0 | 0 | 0 | 0.0 | 0.00 +Comm | 0.036017 | 0.042828 | 0.051558 | 2.7 | 3.82 +Output | 0.0012608 | 0.0025993 | 0.0063775 | 4.3 | 0.23 +Modify | 0.9002 | 0.93095 | 0.99546 | 3.9 | 83.01 +Other | | 0.1451 | | | 12.93 + +Nlocal: 900.000 ave 900 max 900 min +Histogram: 4 0 0 0 0 0 0 0 0 0 +Nghost: 61.0000 ave 61 max 61 min +Histogram: 4 0 0 0 0 0 0 0 0 0 +Neighs: 0.00000 ave 0 max 0 min +Histogram: 4 0 0 0 0 0 0 0 0 0 + +Total # of neighbors = 0 +Ave neighs/atom = 0.0000000 +Neighbor list builds = 0 +Dangerous builds = 0 +Total wall time: 0:00:01 diff --git a/examples/USER/brownian/asphere/log.11May2021.in3d.ellipsoid.g++.1 b/examples/USER/brownian/asphere/log.11May2021.in3d.ellipsoid.g++.1 new file mode 100644 index 0000000000..a193fc98e2 --- /dev/null +++ b/examples/USER/brownian/asphere/log.11May2021.in3d.ellipsoid.g++.1 @@ -0,0 +1,145 @@ +LAMMPS (8 Apr 2021) + using 1 OpenMP thread(s) per MPI task +##### overdamped dynamics of non-interacting ellipsoids in 3D ##### + +variable rng string uniform +variable seed string 198098 +variable temp string 1.0 +variable gamma_r_1 string 2.0 +variable gamma_r_2 string 0.25 +variable gamma_r_3 string 0.1 +variable gamma_t_1 string 5.0 +variable gamma_t_2 string 7.0 +variable gamma_t_3 string 9.0 +variable params string ${rng}_${temp}_${gamma_r_1}_${gamma_r_2}_${gamma_r_3}_${gamma_t_1}_${gamma_t_2}_${gamma_t_3} +variable params string uniform_${temp}_${gamma_r_1}_${gamma_r_2}_${gamma_r_3}_${gamma_t_1}_${gamma_t_2}_${gamma_t_3} +variable params string uniform_1.0_${gamma_r_1}_${gamma_r_2}_${gamma_r_3}_${gamma_t_1}_${gamma_t_2}_${gamma_t_3} +variable params string uniform_1.0_2.0_${gamma_r_2}_${gamma_r_3}_${gamma_t_1}_${gamma_t_2}_${gamma_t_3} +variable params string uniform_1.0_2.0_0.25_${gamma_r_3}_${gamma_t_1}_${gamma_t_2}_${gamma_t_3} +variable params string uniform_1.0_2.0_0.25_0.1_${gamma_t_1}_${gamma_t_2}_${gamma_t_3} +variable params string uniform_1.0_2.0_0.25_0.1_5.0_${gamma_t_2}_${gamma_t_3} +variable params string uniform_1.0_2.0_0.25_0.1_5.0_7.0_${gamma_t_3} +variable params string uniform_1.0_2.0_0.25_0.1_5.0_7.0_9.0 + +units lj +atom_style hybrid dipole ellipsoid +WARNING: Atom style hybrid defines both, per-type and per-atom masses; both must be set, but only per-atom masses will be used (src/atom_vec_hybrid.cpp:156) +dimension 3 +newton off + +lattice sc 0.4 +Lattice spacing in x,y,z = 1.3572088 1.3572088 1.3572088 +region box block -8 8 -8 8 -8 8 +create_box 1 box +Created orthogonal box = (-10.857670 -10.857670 -10.857670) to (10.857670 10.857670 10.857670) + 1 by 1 by 1 MPI processor grid +create_atoms 1 box +Created 4096 atoms + create_atoms CPU = 0.005 seconds +mass * 1.0 +set type * dipole/random ${seed} 1.0 +set type * dipole/random 198098 1.0 +Setting atom values ... + 4096 settings made for dipole/random +set type * shape 3.0 1.0 1.0 +Setting atom values ... + 4096 settings made for shape +set type * quat/random ${seed} +set type * quat/random 198098 +Setting atom values ... + 4096 settings made for quat/random +velocity all create 1.0 1 loop geom + +neighbor 1.0 bin +neigh_modify every 1 delay 1 check yes + +pair_style none + +fix 1 all brownian/asphere ${temp} ${seed} rng ${rng} gamma_r_eigen ${gamma_r_1} ${gamma_r_2} ${gamma_r_3} gamma_t_eigen ${gamma_t_1} ${gamma_t_2} ${gamma_t_3} dipole 1.0 0.0 0.0 +fix 1 all brownian/asphere 1.0 ${seed} rng ${rng} gamma_r_eigen ${gamma_r_1} ${gamma_r_2} ${gamma_r_3} gamma_t_eigen ${gamma_t_1} ${gamma_t_2} ${gamma_t_3} dipole 1.0 0.0 0.0 +fix 1 all brownian/asphere 1.0 198098 rng ${rng} gamma_r_eigen ${gamma_r_1} ${gamma_r_2} ${gamma_r_3} gamma_t_eigen ${gamma_t_1} ${gamma_t_2} ${gamma_t_3} dipole 1.0 0.0 0.0 +fix 1 all brownian/asphere 1.0 198098 rng uniform gamma_r_eigen ${gamma_r_1} ${gamma_r_2} ${gamma_r_3} gamma_t_eigen ${gamma_t_1} ${gamma_t_2} ${gamma_t_3} dipole 1.0 0.0 0.0 +fix 1 all brownian/asphere 1.0 198098 rng uniform gamma_r_eigen 2.0 ${gamma_r_2} ${gamma_r_3} gamma_t_eigen ${gamma_t_1} ${gamma_t_2} ${gamma_t_3} dipole 1.0 0.0 0.0 +fix 1 all brownian/asphere 1.0 198098 rng uniform gamma_r_eigen 2.0 0.25 ${gamma_r_3} gamma_t_eigen ${gamma_t_1} ${gamma_t_2} ${gamma_t_3} dipole 1.0 0.0 0.0 +fix 1 all brownian/asphere 1.0 198098 rng uniform gamma_r_eigen 2.0 0.25 0.1 gamma_t_eigen ${gamma_t_1} ${gamma_t_2} ${gamma_t_3} dipole 1.0 0.0 0.0 +fix 1 all brownian/asphere 1.0 198098 rng uniform gamma_r_eigen 2.0 0.25 0.1 gamma_t_eigen 5.0 ${gamma_t_2} ${gamma_t_3} dipole 1.0 0.0 0.0 +fix 1 all brownian/asphere 1.0 198098 rng uniform gamma_r_eigen 2.0 0.25 0.1 gamma_t_eigen 5.0 7.0 ${gamma_t_3} dipole 1.0 0.0 0.0 +fix 1 all brownian/asphere 1.0 198098 rng uniform gamma_r_eigen 2.0 0.25 0.1 gamma_t_eigen 5.0 7.0 9.0 dipole 1.0 0.0 0.0 + +#initialisation for the main run + +# MSD +compute msd all msd + +thermo_style custom step ke pe c_msd[*] + +#dump 1 all custom 1000 dump_${params}_3d.lammpstrj id type # x y z xu yu zu mux muy muz fx fy fz +#dump_modify 1 first yes sort id + +timestep 0.00001 +thermo 100 + +# main run +run 3000 +WARNING: No pairwise cutoff or binsize set. Atom sorting therefore disabled. (src/atom.cpp:2141) +WARNING: Communication cutoff is 0.0. No ghost atoms will be generated. Atoms may get lost. (src/comm_brick.cpp:167) +Per MPI rank memory allocation (min/avg/max) = 5.219 | 5.219 | 5.219 Mbytes +Step KinEng PotEng c_msd[1] c_msd[2] c_msd[3] c_msd[4] + 0 1.4996338 0 0 0 0 0 + 100 45690.838 0 0.00029994317 0.00029953902 0.00030002809 0.00089951027 + 200 45571.166 0 0.00061376797 0.00060955238 0.00061153551 0.0018348559 + 300 44693.418 0 0.00093058034 0.00089383536 0.00091554588 0.0027399616 + 400 44831.846 0 0.001250227 0.0012230128 0.0012120517 0.0036852914 + 500 45028.015 0 0.0015448869 0.0015339549 0.0014978843 0.0045767262 + 600 45895.442 0 0.0018621952 0.0018169905 0.0018352784 0.0055144641 + 700 45858.744 0 0.0021617097 0.0021137714 0.0021360394 0.0064115206 + 800 45155.215 0 0.002428445 0.0024288837 0.0024516737 0.0073090023 + 900 45427.427 0 0.0027265978 0.0027662531 0.0027329878 0.0082258387 + 1000 45398.166 0 0.0030685345 0.0030805014 0.0029765916 0.0091256275 + 1100 44622.428 0 0.0033766954 0.0033976168 0.0032745406 0.010048853 + 1200 45500.277 0 0.0036410565 0.0036840528 0.0035831659 0.010908275 + 1300 45265.8 0 0.0039143146 0.0039419334 0.0038761633 0.011732411 + 1400 45482.435 0 0.0042006542 0.0043373651 0.004164002 0.012702021 + 1500 45126.629 0 0.0044647379 0.0046021855 0.004487041 0.013553965 + 1600 45178.172 0 0.0047726618 0.0049110287 0.0048012671 0.014484958 + 1700 44918.685 0 0.005104787 0.0052522662 0.0050844375 0.015441491 + 1800 44776.678 0 0.0054395368 0.0056092038 0.0054623875 0.016511128 + 1900 46035.987 0 0.0057735872 0.0059357043 0.0057296009 0.017438892 + 2000 45436.517 0 0.0060837459 0.0063485717 0.0059769119 0.018409229 + 2100 45871.502 0 0.0063736337 0.0066551978 0.0063077439 0.019336575 + 2200 45511.847 0 0.0066419141 0.0069700452 0.0065553318 0.020167291 + 2300 45597.047 0 0.0069251517 0.0073015716 0.0068945654 0.021121289 + 2400 44832.007 0 0.0071894253 0.0076238221 0.0071638554 0.021977103 + 2500 45668.42 0 0.0074351304 0.0079594991 0.0075390719 0.022933701 + 2600 45248.483 0 0.007781496 0.008293944 0.0077956068 0.023871047 + 2700 45308.515 0 0.0080302993 0.0086329679 0.0081457335 0.024809001 + 2800 45637.72 0 0.0083889026 0.0089173198 0.0086032427 0.025909465 + 2900 45909.343 0 0.0087169392 0.009181179 0.0088778569 0.026775975 + 3000 45213.613 0 0.0090508891 0.0094253485 0.0092660321 0.02774227 +Loop time of 4.13937 on 1 procs for 3000 steps with 4096 atoms + +Performance: 626.183 tau/day, 724.749 timesteps/s +99.9% CPU use with 1 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 0 | 0 | 0 | 0.0 | 0.00 +Neigh | 0 | 0 | 0 | 0.0 | 0.00 +Comm | 0.24709 | 0.24709 | 0.24709 | 0.0 | 5.97 +Output | 0.004636 | 0.004636 | 0.004636 | 0.0 | 0.11 +Modify | 3.7604 | 3.7604 | 3.7604 | 0.0 | 90.85 +Other | | 0.1272 | | | 3.07 + +Nlocal: 4096.00 ave 4096 max 4096 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Nghost: 817.000 ave 817 max 817 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Neighs: 0.00000 ave 0 max 0 min +Histogram: 1 0 0 0 0 0 0 0 0 0 + +Total # of neighbors = 0 +Ave neighs/atom = 0.0000000 +Neighbor list builds = 0 +Dangerous builds = 0 +Total wall time: 0:00:04 diff --git a/examples/USER/brownian/asphere/log.11May2021.in3d.ellipsoid.g++.4 b/examples/USER/brownian/asphere/log.11May2021.in3d.ellipsoid.g++.4 new file mode 100644 index 0000000000..7f6d8a81f0 --- /dev/null +++ b/examples/USER/brownian/asphere/log.11May2021.in3d.ellipsoid.g++.4 @@ -0,0 +1,145 @@ +LAMMPS (8 Apr 2021) + using 1 OpenMP thread(s) per MPI task +##### overdamped dynamics of non-interacting ellipsoids in 3D ##### + +variable rng string uniform +variable seed string 198098 +variable temp string 1.0 +variable gamma_r_1 string 2.0 +variable gamma_r_2 string 0.25 +variable gamma_r_3 string 0.1 +variable gamma_t_1 string 5.0 +variable gamma_t_2 string 7.0 +variable gamma_t_3 string 9.0 +variable params string ${rng}_${temp}_${gamma_r_1}_${gamma_r_2}_${gamma_r_3}_${gamma_t_1}_${gamma_t_2}_${gamma_t_3} +variable params string uniform_${temp}_${gamma_r_1}_${gamma_r_2}_${gamma_r_3}_${gamma_t_1}_${gamma_t_2}_${gamma_t_3} +variable params string uniform_1.0_${gamma_r_1}_${gamma_r_2}_${gamma_r_3}_${gamma_t_1}_${gamma_t_2}_${gamma_t_3} +variable params string uniform_1.0_2.0_${gamma_r_2}_${gamma_r_3}_${gamma_t_1}_${gamma_t_2}_${gamma_t_3} +variable params string uniform_1.0_2.0_0.25_${gamma_r_3}_${gamma_t_1}_${gamma_t_2}_${gamma_t_3} +variable params string uniform_1.0_2.0_0.25_0.1_${gamma_t_1}_${gamma_t_2}_${gamma_t_3} +variable params string uniform_1.0_2.0_0.25_0.1_5.0_${gamma_t_2}_${gamma_t_3} +variable params string uniform_1.0_2.0_0.25_0.1_5.0_7.0_${gamma_t_3} +variable params string uniform_1.0_2.0_0.25_0.1_5.0_7.0_9.0 + +units lj +atom_style hybrid dipole ellipsoid +WARNING: Atom style hybrid defines both, per-type and per-atom masses; both must be set, but only per-atom masses will be used (src/atom_vec_hybrid.cpp:156) +dimension 3 +newton off + +lattice sc 0.4 +Lattice spacing in x,y,z = 1.3572088 1.3572088 1.3572088 +region box block -8 8 -8 8 -8 8 +create_box 1 box +Created orthogonal box = (-10.857670 -10.857670 -10.857670) to (10.857670 10.857670 10.857670) + 2 by 1 by 2 MPI processor grid +create_atoms 1 box +Created 4096 atoms + create_atoms CPU = 0.002 seconds +mass * 1.0 +set type * dipole/random ${seed} 1.0 +set type * dipole/random 198098 1.0 +Setting atom values ... + 4096 settings made for dipole/random +set type * shape 3.0 1.0 1.0 +Setting atom values ... + 4096 settings made for shape +set type * quat/random ${seed} +set type * quat/random 198098 +Setting atom values ... + 4096 settings made for quat/random +velocity all create 1.0 1 loop geom + +neighbor 1.0 bin +neigh_modify every 1 delay 1 check yes + +pair_style none + +fix 1 all brownian/asphere ${temp} ${seed} rng ${rng} gamma_r_eigen ${gamma_r_1} ${gamma_r_2} ${gamma_r_3} gamma_t_eigen ${gamma_t_1} ${gamma_t_2} ${gamma_t_3} dipole 1.0 0.0 0.0 +fix 1 all brownian/asphere 1.0 ${seed} rng ${rng} gamma_r_eigen ${gamma_r_1} ${gamma_r_2} ${gamma_r_3} gamma_t_eigen ${gamma_t_1} ${gamma_t_2} ${gamma_t_3} dipole 1.0 0.0 0.0 +fix 1 all brownian/asphere 1.0 198098 rng ${rng} gamma_r_eigen ${gamma_r_1} ${gamma_r_2} ${gamma_r_3} gamma_t_eigen ${gamma_t_1} ${gamma_t_2} ${gamma_t_3} dipole 1.0 0.0 0.0 +fix 1 all brownian/asphere 1.0 198098 rng uniform gamma_r_eigen ${gamma_r_1} ${gamma_r_2} ${gamma_r_3} gamma_t_eigen ${gamma_t_1} ${gamma_t_2} ${gamma_t_3} dipole 1.0 0.0 0.0 +fix 1 all brownian/asphere 1.0 198098 rng uniform gamma_r_eigen 2.0 ${gamma_r_2} ${gamma_r_3} gamma_t_eigen ${gamma_t_1} ${gamma_t_2} ${gamma_t_3} dipole 1.0 0.0 0.0 +fix 1 all brownian/asphere 1.0 198098 rng uniform gamma_r_eigen 2.0 0.25 ${gamma_r_3} gamma_t_eigen ${gamma_t_1} ${gamma_t_2} ${gamma_t_3} dipole 1.0 0.0 0.0 +fix 1 all brownian/asphere 1.0 198098 rng uniform gamma_r_eigen 2.0 0.25 0.1 gamma_t_eigen ${gamma_t_1} ${gamma_t_2} ${gamma_t_3} dipole 1.0 0.0 0.0 +fix 1 all brownian/asphere 1.0 198098 rng uniform gamma_r_eigen 2.0 0.25 0.1 gamma_t_eigen 5.0 ${gamma_t_2} ${gamma_t_3} dipole 1.0 0.0 0.0 +fix 1 all brownian/asphere 1.0 198098 rng uniform gamma_r_eigen 2.0 0.25 0.1 gamma_t_eigen 5.0 7.0 ${gamma_t_3} dipole 1.0 0.0 0.0 +fix 1 all brownian/asphere 1.0 198098 rng uniform gamma_r_eigen 2.0 0.25 0.1 gamma_t_eigen 5.0 7.0 9.0 dipole 1.0 0.0 0.0 + +#initialisation for the main run + +# MSD +compute msd all msd + +thermo_style custom step ke pe c_msd[*] + +#dump 1 all custom 1000 dump_${params}_3d.lammpstrj id type # x y z xu yu zu mux muy muz fx fy fz +#dump_modify 1 first yes sort id + +timestep 0.00001 +thermo 100 + +# main run +run 3000 +WARNING: No pairwise cutoff or binsize set. Atom sorting therefore disabled. (src/atom.cpp:2141) +WARNING: Communication cutoff is 0.0. No ghost atoms will be generated. Atoms may get lost. (src/comm_brick.cpp:167) +Per MPI rank memory allocation (min/avg/max) = 5.152 | 5.152 | 5.152 Mbytes +Step KinEng PotEng c_msd[1] c_msd[2] c_msd[3] c_msd[4] + 0 1.4996338 0 0 0 0 0 + 100 45236.508 0 0.00030817418 0.00030717742 0.0003019227 0.0009172743 + 200 45564.566 0 0.00062027526 0.00062110132 0.0006080391 0.0018494157 + 300 46232.801 0 0.00091155216 0.00094473459 0.00093009391 0.0027863807 + 400 45250.414 0 0.0011980791 0.0012538262 0.0012201461 0.0036720513 + 500 45217.133 0 0.0015186813 0.0015752994 0.001509437 0.0046034177 + 600 45531.276 0 0.0018194588 0.0019243758 0.0018209246 0.0055647592 + 700 44834.624 0 0.0021277747 0.0022417115 0.0021352036 0.0065046898 + 800 45413.998 0 0.0024558838 0.0025741787 0.0024088704 0.0074389329 + 900 45668.624 0 0.0027366171 0.002858242 0.0027580782 0.0083529374 + 1000 45809.223 0 0.0030331425 0.003186293 0.0030414906 0.0092609261 + 1100 45193.019 0 0.0033199824 0.0034668659 0.003298885 0.010085733 + 1200 44522.927 0 0.0036503132 0.0037490684 0.0036089852 0.011008367 + 1300 45214.567 0 0.0039958617 0.0040881934 0.0038709079 0.011954963 + 1400 45217.997 0 0.004276499 0.0044624985 0.0041104891 0.012849487 + 1500 45497.171 0 0.0045943272 0.0047116875 0.0044113504 0.013717365 + 1600 45905.187 0 0.0049004996 0.0049982014 0.0047394999 0.014638201 + 1700 45551.346 0 0.0051540939 0.0053187249 0.0050861052 0.015558924 + 1800 45347.782 0 0.0054101891 0.0056306 0.0053515873 0.016392376 + 1900 45107.895 0 0.005743705 0.0059584896 0.0056220384 0.017324233 + 2000 45043.389 0 0.0059803588 0.006230449 0.005911555 0.018122363 + 2100 45433.293 0 0.0062610364 0.0066140744 0.0062152977 0.019090408 + 2200 45804.217 0 0.0064995183 0.0068831274 0.0064971789 0.019879825 + 2300 45697.516 0 0.0067910846 0.0071845673 0.0068046192 0.020780271 + 2400 45447.422 0 0.0071022706 0.0074743709 0.0070983185 0.02167496 + 2500 45395.18 0 0.0073817023 0.0077467991 0.0074263196 0.022554821 + 2600 45943.044 0 0.0075953233 0.007997707 0.0076508583 0.023243889 + 2700 45859.978 0 0.0079082128 0.0082090043 0.0078853376 0.024002555 + 2800 45822.007 0 0.0082607534 0.0084510061 0.0081985549 0.024910314 + 2900 45438.456 0 0.0085958203 0.0088807705 0.0084755353 0.025952126 + 3000 45060.957 0 0.0089017992 0.0090966159 0.0086718875 0.026670303 +Loop time of 1.23282 on 4 procs for 3000 steps with 4096 atoms + +Performance: 2102.502 tau/day, 2433.452 timesteps/s +97.9% CPU use with 4 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 0 | 0 | 0 | 0.0 | 0.00 +Neigh | 8.5831e-06 | 8.5831e-06 | 8.5831e-06 | 0.0 | 0.00 +Comm | 0.10931 | 0.11473 | 0.11748 | 0.9 | 9.31 +Output | 0.001375 | 0.0018924 | 0.0034099 | 2.0 | 0.15 +Modify | 0.97744 | 0.99158 | 1.0089 | 1.3 | 80.43 +Other | | 0.1246 | | | 10.11 + +Nlocal: 1024.00 ave 1035 max 1016 min +Histogram: 2 0 0 0 0 0 1 0 0 1 +Nghost: 0.00000 ave 0 max 0 min +Histogram: 4 0 0 0 0 0 0 0 0 0 +Neighs: 0.00000 ave 0 max 0 min +Histogram: 4 0 0 0 0 0 0 0 0 0 + +Total # of neighbors = 0 +Ave neighs/atom = 0.0000000 +Neighbor list builds = 1 +Dangerous builds = 0 +Total wall time: 0:00:01 diff --git a/examples/USER/brownian/point/in2d.point b/examples/USER/brownian/point/in2d.point new file mode 100644 index 0000000000..bf790f8fc2 --- /dev/null +++ b/examples/USER/brownian/point/in2d.point @@ -0,0 +1,43 @@ +##### dynamics of non-interacting point particles in 2D ##### + +variable rng string gaussian +variable seed string 198098 +variable temp string 5.0 +variable gamma_t string 1.0 +variable params string ${rng}_${temp}_${gamma_t} + +units lj +atom_style atomic +dimension 2 +newton off + +lattice sq 0.4 +region box block -30 30 -30 30 -0.2 0.2 +create_box 1 box +create_atoms 1 box +mass * 1.0 +velocity all create 1.0 1 loop geom + +neighbor 1.0 bin +neigh_modify every 1 delay 1 check yes + +pair_style none + +fix 1 all brownian ${temp} ${seed} rng ${rng} gamma_t ${gamma_t} + +#initialisation for the main run + +# MSD +compute msd all msd + +thermo_style custom step ke pe c_msd[*] + +#dump 1 all custom 1000 dump_${params}_2d.lammpstrj id type & +# x y z xu yu zu fx fy fz +#dump_modify 1 first yes sort id + +timestep 0.00001 +thermo 100 + +# main run +run 3000 diff --git a/examples/USER/brownian/point/in3d.point b/examples/USER/brownian/point/in3d.point new file mode 100644 index 0000000000..974f08866d --- /dev/null +++ b/examples/USER/brownian/point/in3d.point @@ -0,0 +1,44 @@ +##### overdamped dynamics of non-interacting point particles in 3D ##### + +variable rng string gaussian +variable seed string 198098 +variable temp string 5.0 +variable gamma_t string 1.0 +variable params string ${rng}_${temp}_${gamma_t} + +units lj +atom_style atomic +dimension 3 +newton off + +lattice sc 0.4 +region box block -8 8 -8 8 -8 8 +create_box 1 box +create_atoms 1 box +mass * 1.0 +velocity all create 1.0 1 loop geom + +neighbor 1.0 bin +neigh_modify every 1 delay 1 check yes + +pair_style none + +fix 1 all brownian ${temp} ${seed} rng ${rng} gamma_t ${gamma_t} + +#initialisation for the main run + +# MSD +compute msd all msd + +thermo_style custom step ke pe c_msd[*] + +#dump 1 all custom 1000 dump_${params}_3d.lammpstrj id type & +# x y z xu yu zu fx fy fz +#dump_modify 1 first yes sort id + +timestep 0.00001 +thermo 100 + +# main run +run 3000 + diff --git a/examples/USER/brownian/point/log.11May2021.in2d.point.g++.1 b/examples/USER/brownian/point/log.11May2021.in2d.point.g++.1 new file mode 100644 index 0000000000..b875ccedad --- /dev/null +++ b/examples/USER/brownian/point/log.11May2021.in2d.point.g++.1 @@ -0,0 +1,119 @@ +LAMMPS (8 Apr 2021) + using 1 OpenMP thread(s) per MPI task +##### dynamics of non-interacting point particles in 2D ##### + +variable rng string gaussian +variable seed string 198098 +variable temp string 5.0 +variable gamma_t string 1.0 +variable params string ${rng}_${temp}_${gamma_t} +variable params string gaussian_${temp}_${gamma_t} +variable params string gaussian_5.0_${gamma_t} +variable params string gaussian_5.0_1.0 + +units lj +atom_style atomic +dimension 2 +newton off + +lattice sq 0.4 +Lattice spacing in x,y,z = 1.5811388 1.5811388 1.5811388 +region box block -30 30 -30 30 -0.2 0.2 +create_box 1 box +Created orthogonal box = (-47.434165 -47.434165 -0.31622777) to (47.434165 47.434165 0.31622777) + 1 by 1 by 1 MPI processor grid +create_atoms 1 box +Created 3600 atoms + create_atoms CPU = 0.003 seconds +mass * 1.0 +velocity all create 1.0 1 loop geom + +neighbor 1.0 bin +neigh_modify every 1 delay 1 check yes + +pair_style none + +fix 1 all brownian ${temp} ${seed} rng ${rng} gamma_t ${gamma_t} +fix 1 all brownian 5.0 ${seed} rng ${rng} gamma_t ${gamma_t} +fix 1 all brownian 5.0 198098 rng ${rng} gamma_t ${gamma_t} +fix 1 all brownian 5.0 198098 rng gaussian gamma_t ${gamma_t} +fix 1 all brownian 5.0 198098 rng gaussian gamma_t 1.0 + +#initialisation for the main run + +# MSD +compute msd all msd + +thermo_style custom step ke pe c_msd[*] + +#dump 1 all custom 1000 dump_${params}_2d.lammpstrj id type # x y z xu yu zu fx fy fz +#dump_modify 1 first yes sort id + +timestep 0.00001 +thermo 100 + +# main run +run 3000 +WARNING: No pairwise cutoff or binsize set. Atom sorting therefore disabled. (src/atom.cpp:2141) +WARNING: Communication cutoff is 0.0. No ghost atoms will be generated. Atoms may get lost. (src/comm_brick.cpp:167) +Per MPI rank memory allocation (min/avg/max) = 2.664 | 2.664 | 2.664 Mbytes +Step KinEng PotEng c_msd[1] c_msd[2] c_msd[3] c_msd[4] + 0 0.99972222 0 0 0 0 0 + 100 1022861.2 0 0.010252464 0.0095481044 0 0.019800568 + 200 986781.19 0 0.020552091 0.019485252 0 0.040037343 + 300 1030219 0 0.030642552 0.028377678 0 0.05902023 + 400 1003322.5 0 0.040610693 0.038179284 0 0.078789978 + 500 989343.12 0 0.049978908 0.047445856 0 0.097424764 + 600 1029781.3 0 0.059551719 0.057941149 0 0.11749287 + 700 999447.72 0 0.06979546 0.067552325 0 0.13734778 + 800 995373.97 0 0.080049251 0.078006344 0 0.1580556 + 900 1011991.4 0 0.089753134 0.087065214 0 0.17681835 + 1000 1006017.1 0 0.10041092 0.097934217 0 0.19834514 + 1100 997762.63 0 0.11229742 0.10841547 0 0.22071289 + 1200 1011707.8 0 0.12006388 0.1190115 0 0.23907538 + 1300 1012099.1 0 0.13097486 0.12996632 0 0.26094117 + 1400 997602.43 0 0.14345778 0.13830585 0 0.28176362 + 1500 1005358.1 0 0.15441686 0.14927539 0 0.30369225 + 1600 1007081.8 0 0.16496828 0.15936363 0 0.3243319 + 1700 990284.9 0 0.1747286 0.16818246 0 0.34291106 + 1800 969006.97 0 0.18228778 0.17972813 0 0.3620159 + 1900 998066.69 0 0.19338277 0.19226121 0 0.38564397 + 2000 972300.66 0 0.20352485 0.20145928 0 0.40498413 + 2100 985025.88 0 0.21283854 0.21090075 0 0.42373929 + 2200 1010964.6 0 0.22279055 0.22110734 0 0.44389789 + 2300 975819.44 0 0.23128131 0.23226488 0 0.46354619 + 2400 977043.53 0 0.24284105 0.24301689 0 0.48585794 + 2500 969708.21 0 0.25415238 0.25354284 0 0.50769522 + 2600 981969.5 0 0.26457173 0.26318018 0 0.52775192 + 2700 987261.1 0 0.27497004 0.27761213 0 0.55258218 + 2800 1005751.1 0 0.28530448 0.28715428 0 0.57245876 + 2900 975930.11 0 0.29394811 0.29896948 0 0.59291759 + 3000 997388.08 0 0.30674701 0.31193573 0 0.61868274 +Loop time of 1.501 on 1 procs for 3000 steps with 3600 atoms + +Performance: 1726.852 tau/day, 1998.672 timesteps/s +99.9% CPU use with 1 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 0 | 0 | 0 | 0.0 | 0.00 +Neigh | 0.00059271 | 0.00059271 | 0.00059271 | 0.0 | 0.04 +Comm | 0.0055437 | 0.0055437 | 0.0055437 | 0.0 | 0.37 +Output | 0.0039999 | 0.0039999 | 0.0039999 | 0.0 | 0.27 +Modify | 1.3852 | 1.3852 | 1.3852 | 0.0 | 92.28 +Other | | 0.1057 | | | 7.04 + +Nlocal: 3600.00 ave 3600 max 3600 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Nghost: 0.00000 ave 0 max 0 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Neighs: 0.00000 ave 0 max 0 min +Histogram: 1 0 0 0 0 0 0 0 0 0 + +Total # of neighbors = 0 +Ave neighs/atom = 0.0000000 +Neighbor list builds = 21 +Dangerous builds = 0 + +Total wall time: 0:00:01 diff --git a/examples/USER/brownian/point/log.11May2021.in2d.point.g++.4 b/examples/USER/brownian/point/log.11May2021.in2d.point.g++.4 new file mode 100644 index 0000000000..d12d71118e --- /dev/null +++ b/examples/USER/brownian/point/log.11May2021.in2d.point.g++.4 @@ -0,0 +1,119 @@ +LAMMPS (8 Apr 2021) + using 1 OpenMP thread(s) per MPI task +##### dynamics of non-interacting point particles in 2D ##### + +variable rng string gaussian +variable seed string 198098 +variable temp string 5.0 +variable gamma_t string 1.0 +variable params string ${rng}_${temp}_${gamma_t} +variable params string gaussian_${temp}_${gamma_t} +variable params string gaussian_5.0_${gamma_t} +variable params string gaussian_5.0_1.0 + +units lj +atom_style atomic +dimension 2 +newton off + +lattice sq 0.4 +Lattice spacing in x,y,z = 1.5811388 1.5811388 1.5811388 +region box block -30 30 -30 30 -0.2 0.2 +create_box 1 box +Created orthogonal box = (-47.434165 -47.434165 -0.31622777) to (47.434165 47.434165 0.31622777) + 2 by 2 by 1 MPI processor grid +create_atoms 1 box +Created 3600 atoms + create_atoms CPU = 0.001 seconds +mass * 1.0 +velocity all create 1.0 1 loop geom + +neighbor 1.0 bin +neigh_modify every 1 delay 1 check yes + +pair_style none + +fix 1 all brownian ${temp} ${seed} rng ${rng} gamma_t ${gamma_t} +fix 1 all brownian 5.0 ${seed} rng ${rng} gamma_t ${gamma_t} +fix 1 all brownian 5.0 198098 rng ${rng} gamma_t ${gamma_t} +fix 1 all brownian 5.0 198098 rng gaussian gamma_t ${gamma_t} +fix 1 all brownian 5.0 198098 rng gaussian gamma_t 1.0 + +#initialisation for the main run + +# MSD +compute msd all msd + +thermo_style custom step ke pe c_msd[*] + +#dump 1 all custom 1000 dump_${params}_2d.lammpstrj id type # x y z xu yu zu fx fy fz +#dump_modify 1 first yes sort id + +timestep 0.00001 +thermo 100 + +# main run +run 3000 +WARNING: No pairwise cutoff or binsize set. Atom sorting therefore disabled. (src/atom.cpp:2141) +WARNING: Communication cutoff is 0.0. No ghost atoms will be generated. Atoms may get lost. (src/comm_brick.cpp:167) +Per MPI rank memory allocation (min/avg/max) = 2.664 | 2.664 | 2.664 Mbytes +Step KinEng PotEng c_msd[1] c_msd[2] c_msd[3] c_msd[4] + 0 0.99972222 0 0 0 0 0 + 100 1017972.1 0 0.010094052 0.0097502899 0 0.019844342 + 200 1004552.1 0 0.020125116 0.01957629 0 0.039701406 + 300 1017712.9 0 0.030271373 0.029411656 0 0.059683029 + 400 1016693.8 0 0.040610061 0.038605869 0 0.07921593 + 500 999527.84 0 0.049451389 0.049042225 0 0.098493614 + 600 961157.92 0 0.059691948 0.059033176 0 0.11872512 + 700 1006804.9 0 0.071205977 0.069972106 0 0.14117808 + 800 1007321.8 0 0.081136977 0.079825976 0 0.16096295 + 900 1002801.7 0 0.091236148 0.090833816 0 0.18206996 + 1000 1010134.7 0 0.10091362 0.10023906 0 0.20115269 + 1100 990246.55 0 0.1118367 0.11141049 0 0.22324719 + 1200 1010555.5 0 0.12091736 0.12355456 0 0.24447192 + 1300 997117.19 0 0.13099592 0.13292775 0 0.26392367 + 1400 1020817.1 0 0.14167961 0.14172898 0 0.28340859 + 1500 1015048.1 0 0.15225884 0.15162948 0 0.30388833 + 1600 990291.98 0 0.16460973 0.16251919 0 0.32712891 + 1700 980848.58 0 0.17380313 0.17351201 0 0.34731513 + 1800 1000673.8 0 0.18383991 0.18175453 0 0.36559445 + 1900 1009388.9 0 0.19411523 0.19367453 0 0.38778976 + 2000 1005935.9 0 0.2015342 0.20585359 0 0.40738779 + 2100 985500.56 0 0.21161056 0.21238463 0 0.42399519 + 2200 997241.34 0 0.21841986 0.22117922 0 0.43959908 + 2300 1011672.3 0 0.22688099 0.23155741 0 0.4584384 + 2400 989837.68 0 0.23849839 0.24219 0 0.48068839 + 2500 1035706.8 0 0.24541408 0.24947563 0 0.49488971 + 2600 992370.08 0 0.25537803 0.25758332 0 0.51296135 + 2700 990586.56 0 0.26542605 0.26762286 0 0.53304892 + 2800 1002767.3 0 0.27570392 0.27874972 0 0.55445363 + 2900 995307.27 0 0.28580946 0.29115624 0 0.5769657 + 3000 1024317.7 0 0.29493208 0.30208924 0 0.59702132 +Loop time of 0.413047 on 4 procs for 3000 steps with 3600 atoms + +Performance: 6275.312 tau/day, 7263.093 timesteps/s +98.0% CPU use with 4 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 0 | 0 | 0 | 0.0 | 0.00 +Neigh | 0.00016236 | 0.00016338 | 0.00016403 | 0.0 | 0.04 +Comm | 0.0026367 | 0.0030084 | 0.0031497 | 0.4 | 0.73 +Output | 0.0011849 | 0.0013574 | 0.0018065 | 0.7 | 0.33 +Modify | 0.34447 | 0.35223 | 0.36357 | 1.2 | 85.28 +Other | | 0.05629 | | | 13.63 + +Nlocal: 900.000 ave 906 max 891 min +Histogram: 1 0 0 0 0 0 1 0 1 1 +Nghost: 0.00000 ave 0 max 0 min +Histogram: 4 0 0 0 0 0 0 0 0 0 +Neighs: 0.00000 ave 0 max 0 min +Histogram: 4 0 0 0 0 0 0 0 0 0 + +Total # of neighbors = 0 +Ave neighs/atom = 0.0000000 +Neighbor list builds = 22 +Dangerous builds = 0 + +Total wall time: 0:00:00 diff --git a/examples/USER/brownian/point/log.11May2021.in3d.point.g++.1 b/examples/USER/brownian/point/log.11May2021.in3d.point.g++.1 new file mode 100644 index 0000000000..febb7657e7 --- /dev/null +++ b/examples/USER/brownian/point/log.11May2021.in3d.point.g++.1 @@ -0,0 +1,119 @@ +LAMMPS (8 Apr 2021) + using 1 OpenMP thread(s) per MPI task +##### overdamped dynamics of non-interacting point particles in 3D ##### + +variable rng string gaussian +variable seed string 198098 +variable temp string 5.0 +variable gamma_t string 1.0 +variable params string ${rng}_${temp}_${gamma_t} +variable params string gaussian_${temp}_${gamma_t} +variable params string gaussian_5.0_${gamma_t} +variable params string gaussian_5.0_1.0 + +units lj +atom_style atomic +dimension 3 +newton off + +lattice sc 0.4 +Lattice spacing in x,y,z = 1.3572088 1.3572088 1.3572088 +region box block -8 8 -8 8 -8 8 +create_box 1 box +Created orthogonal box = (-10.857670 -10.857670 -10.857670) to (10.857670 10.857670 10.857670) + 1 by 1 by 1 MPI processor grid +create_atoms 1 box +Created 4096 atoms + create_atoms CPU = 0.002 seconds +mass * 1.0 +velocity all create 1.0 1 loop geom + +neighbor 1.0 bin +neigh_modify every 1 delay 1 check yes + +pair_style none + +fix 1 all brownian ${temp} ${seed} rng ${rng} gamma_t ${gamma_t} +fix 1 all brownian 5.0 ${seed} rng ${rng} gamma_t ${gamma_t} +fix 1 all brownian 5.0 198098 rng ${rng} gamma_t ${gamma_t} +fix 1 all brownian 5.0 198098 rng gaussian gamma_t ${gamma_t} +fix 1 all brownian 5.0 198098 rng gaussian gamma_t 1.0 + +#initialisation for the main run + +# MSD +compute msd all msd + +thermo_style custom step ke pe c_msd[*] + +#dump 1 all custom 1000 dump_${params}_3d.lammpstrj id type # x y z xu yu zu fx fy fz +#dump_modify 1 first yes sort id + +timestep 0.00001 +thermo 100 + +# main run +run 3000 +WARNING: No pairwise cutoff or binsize set. Atom sorting therefore disabled. (src/atom.cpp:2141) +WARNING: Communication cutoff is 0.0. No ghost atoms will be generated. Atoms may get lost. (src/comm_brick.cpp:167) +Per MPI rank memory allocation (min/avg/max) = 2.694 | 2.694 | 2.694 Mbytes +Step KinEng PotEng c_msd[1] c_msd[2] c_msd[3] c_msd[4] + 0 1.4996338 0 0 0 0 0 + 100 1500286.3 0 0.0098123603 0.010352169 0.010242435 0.030406964 + 200 1488308 0 0.019934427 0.019968198 0.020471735 0.06037436 + 300 1484472.4 0 0.029397156 0.030749312 0.030121294 0.090267762 + 400 1517938.7 0 0.039217504 0.041440617 0.040512943 0.12117106 + 500 1492769.5 0 0.04890343 0.051561801 0.050614941 0.15108017 + 600 1510159.6 0 0.059770181 0.061650364 0.061298117 0.18271866 + 700 1485424.1 0 0.070537955 0.071144877 0.071141546 0.21282438 + 800 1496377.2 0 0.081291995 0.082546059 0.080653381 0.24449144 + 900 1484409.1 0 0.090940427 0.093298981 0.091328056 0.27556746 + 1000 1503322.4 0 0.10176921 0.10246052 0.10151773 0.30574747 + 1100 1503322.4 0 0.11295993 0.11052632 0.11053406 0.33402031 + 1200 1489236.2 0 0.12509723 0.11961982 0.12146498 0.36618203 + 1300 1476050.3 0 0.13449034 0.12941323 0.1309765 0.39488007 + 1400 1520818.7 0 0.14613571 0.13788044 0.14083944 0.42485558 + 1500 1498936.4 0 0.15752286 0.15057712 0.15063399 0.45873397 + 1600 1507524.1 0 0.16793678 0.16095681 0.16063531 0.4895289 + 1700 1480581.2 0 0.17748019 0.172614 0.16922383 0.51931802 + 1800 1505353.6 0 0.18850931 0.18304171 0.18063119 0.55218221 + 1900 1491234.7 0 0.19836402 0.19306339 0.1929707 0.58439811 + 2000 1519868.8 0 0.20698191 0.20211344 0.20328302 0.61237838 + 2100 1493919.5 0 0.21453524 0.21186097 0.21423293 0.64062914 + 2200 1517098.6 0 0.2257338 0.22381647 0.22474081 0.67429108 + 2300 1481270.7 0 0.23499747 0.23348379 0.23498244 0.70346369 + 2400 1495445.1 0 0.24535894 0.24290239 0.24229161 0.73055293 + 2500 1522839.3 0 0.25695938 0.25109669 0.25214541 0.76020148 + 2600 1518697.4 0 0.26680819 0.26120216 0.2604112 0.78842155 + 2700 1529283.1 0 0.27524422 0.26942681 0.27148042 0.81615146 + 2800 1500557.5 0 0.28436226 0.27957592 0.27935619 0.84329437 + 2900 1509711.1 0 0.2948528 0.28562401 0.29055956 0.87103637 + 3000 1522712.8 0 0.30347033 0.2975063 0.30121685 0.90219348 +Loop time of 2.35056 on 1 procs for 3000 steps with 4096 atoms + +Performance: 1102.718 tau/day, 1276.293 timesteps/s +99.9% CPU use with 1 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 0 | 0 | 0 | 0.0 | 0.00 +Neigh | 0.00077772 | 0.00077772 | 0.00077772 | 0.0 | 0.03 +Comm | 0.010985 | 0.010985 | 0.010985 | 0.0 | 0.47 +Output | 0.0045807 | 0.0045807 | 0.0045807 | 0.0 | 0.19 +Modify | 2.2116 | 2.2116 | 2.2116 | 0.0 | 94.09 +Other | | 0.1226 | | | 5.22 + +Nlocal: 4096.00 ave 4096 max 4096 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Nghost: 0.00000 ave 0 max 0 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Neighs: 0.00000 ave 0 max 0 min +Histogram: 1 0 0 0 0 0 0 0 0 0 + +Total # of neighbors = 0 +Ave neighs/atom = 0.0000000 +Neighbor list builds = 24 +Dangerous builds = 0 + +Total wall time: 0:00:02 diff --git a/examples/USER/brownian/point/log.11May2021.in3d.point.g++.4 b/examples/USER/brownian/point/log.11May2021.in3d.point.g++.4 new file mode 100644 index 0000000000..6a41a12f0f --- /dev/null +++ b/examples/USER/brownian/point/log.11May2021.in3d.point.g++.4 @@ -0,0 +1,119 @@ +LAMMPS (8 Apr 2021) + using 1 OpenMP thread(s) per MPI task +##### overdamped dynamics of non-interacting point particles in 3D ##### + +variable rng string gaussian +variable seed string 198098 +variable temp string 5.0 +variable gamma_t string 1.0 +variable params string ${rng}_${temp}_${gamma_t} +variable params string gaussian_${temp}_${gamma_t} +variable params string gaussian_5.0_${gamma_t} +variable params string gaussian_5.0_1.0 + +units lj +atom_style atomic +dimension 3 +newton off + +lattice sc 0.4 +Lattice spacing in x,y,z = 1.3572088 1.3572088 1.3572088 +region box block -8 8 -8 8 -8 8 +create_box 1 box +Created orthogonal box = (-10.857670 -10.857670 -10.857670) to (10.857670 10.857670 10.857670) + 2 by 1 by 2 MPI processor grid +create_atoms 1 box +Created 4096 atoms + create_atoms CPU = 0.001 seconds +mass * 1.0 +velocity all create 1.0 1 loop geom + +neighbor 1.0 bin +neigh_modify every 1 delay 1 check yes + +pair_style none + +fix 1 all brownian ${temp} ${seed} rng ${rng} gamma_t ${gamma_t} +fix 1 all brownian 5.0 ${seed} rng ${rng} gamma_t ${gamma_t} +fix 1 all brownian 5.0 198098 rng ${rng} gamma_t ${gamma_t} +fix 1 all brownian 5.0 198098 rng gaussian gamma_t ${gamma_t} +fix 1 all brownian 5.0 198098 rng gaussian gamma_t 1.0 + +#initialisation for the main run + +# MSD +compute msd all msd + +thermo_style custom step ke pe c_msd[*] + +#dump 1 all custom 1000 dump_${params}_3d.lammpstrj id type # x y z xu yu zu fx fy fz +#dump_modify 1 first yes sort id + +timestep 0.00001 +thermo 100 + +# main run +run 3000 +WARNING: No pairwise cutoff or binsize set. Atom sorting therefore disabled. (src/atom.cpp:2141) +WARNING: Communication cutoff is 0.0. No ghost atoms will be generated. Atoms may get lost. (src/comm_brick.cpp:167) +Per MPI rank memory allocation (min/avg/max) = 2.672 | 2.672 | 2.672 Mbytes +Step KinEng PotEng c_msd[1] c_msd[2] c_msd[3] c_msd[4] + 0 1.4996338 0 0 0 0 0 + 100 1515328.2 0 0.010465453 0.010044629 0.0097242319 0.030234314 + 200 1510820.8 0 0.020658886 0.019954762 0.020008864 0.060622512 + 300 1482006.5 0 0.030402195 0.029802874 0.030047586 0.090252655 + 400 1492228.5 0 0.039622543 0.038899144 0.040381854 0.11890354 + 500 1494985.5 0 0.050523465 0.050022913 0.050186478 0.15073286 + 600 1516047.4 0 0.061111845 0.061433818 0.059195364 0.18174103 + 700 1510021.8 0 0.071636778 0.072829755 0.06946406 0.21393059 + 800 1505964.7 0 0.08240965 0.08433785 0.078799851 0.24554735 + 900 1491035.9 0 0.093659937 0.094517749 0.08812559 0.27630328 + 1000 1516599.6 0 0.10436496 0.10431759 0.097480868 0.30616342 + 1100 1495170.3 0 0.11468757 0.111397 0.1069763 0.33306087 + 1200 1500630.6 0 0.12360977 0.12264534 0.11583999 0.3620951 + 1300 1474889.5 0 0.13432447 0.13471694 0.12702491 0.39606632 + 1400 1487145.8 0 0.14573239 0.14431493 0.13669403 0.42674135 + 1500 1519496.7 0 0.15610742 0.15505416 0.14600182 0.4571634 + 1600 1525674.1 0 0.16728653 0.1649354 0.15562133 0.48784325 + 1700 1540725.4 0 0.17846447 0.17666562 0.16531781 0.52044791 + 1800 1512334.8 0 0.18872753 0.18538847 0.17450009 0.54861609 + 1900 1498371.4 0 0.19688928 0.19333299 0.18581712 0.5760394 + 2000 1546459.4 0 0.20955053 0.20243854 0.19613897 0.60812803 + 2100 1509712.9 0 0.21922567 0.20940597 0.20567239 0.63430404 + 2200 1509630.4 0 0.23067999 0.21856734 0.21619911 0.66544645 + 2300 1483929.1 0 0.24160803 0.231048 0.22617193 0.69882797 + 2400 1488492.1 0 0.25399491 0.24082678 0.23972356 0.73454526 + 2500 1508107.9 0 0.26608734 0.25316913 0.2486814 0.76793787 + 2600 1511952.1 0 0.27523956 0.2623673 0.25706539 0.79467225 + 2700 1488888.8 0 0.28518299 0.27425585 0.26728622 0.82672506 + 2800 1515428.4 0 0.29595429 0.28589969 0.27781327 0.85966725 + 2900 1504312.1 0 0.30393798 0.29533034 0.28725362 0.88652194 + 3000 1521521.3 0 0.31445132 0.30117607 0.29959324 0.91522062 +Loop time of 0.708196 on 4 procs for 3000 steps with 4096 atoms + +Performance: 3660.004 tau/day, 4236.115 timesteps/s +97.2% CPU use with 4 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 0 | 0 | 0 | 0.0 | 0.00 +Neigh | 0.00020647 | 0.00021023 | 0.0002141 | 0.0 | 0.03 +Comm | 0.0045607 | 0.0050649 | 0.0053098 | 0.4 | 0.72 +Output | 0.0013759 | 0.002265 | 0.0037355 | 1.9 | 0.32 +Modify | 0.57353 | 0.58931 | 0.6109 | 1.8 | 83.21 +Other | | 0.1113 | | | 15.72 + +Nlocal: 1024.00 ave 1043 max 1001 min +Histogram: 1 0 0 0 0 2 0 0 0 1 +Nghost: 0.00000 ave 0 max 0 min +Histogram: 4 0 0 0 0 0 0 0 0 0 +Neighs: 0.00000 ave 0 max 0 min +Histogram: 4 0 0 0 0 0 0 0 0 0 + +Total # of neighbors = 0 +Ave neighs/atom = 0.0000000 +Neighbor list builds = 25 +Dangerous builds = 0 + +Total wall time: 0:00:00 diff --git a/examples/USER/brownian/sphere/in2d.sphere b/examples/USER/brownian/sphere/in2d.sphere new file mode 100644 index 0000000000..a194f967cc --- /dev/null +++ b/examples/USER/brownian/sphere/in2d.sphere @@ -0,0 +1,45 @@ +##### overdamped dynamics of a sphere (with dipole attached to it) in 2D ##### + +variable rng string uniform +variable seed string 198098 +variable temp string 1.0 +variable gamma_t string 5.0 +variable gamma_r string 0.7 +variable params string ${rng}_${temp}_${gamma_r}_${gamma_t} + +units lj +atom_style hybrid dipole sphere +dimension 2 +newton off + +lattice sq 0.4 +region box block -30 30 -30 30 -0.2 0.2 +create_box 1 box +create_atoms 1 box +mass * 1.0 +set type * dipole/random ${seed} 1.0 +velocity all create 1.0 1 loop geom + +neighbor 1.0 bin +neigh_modify every 1 delay 1 check yes + +pair_style none + +fix 1 all brownian/sphere ${temp} ${seed} rng ${rng} gamma_r ${gamma_r} gamma_t ${gamma_t} + +#initialisation for the main run + +# MSD +compute msd all msd + +thermo_style custom step ke pe c_msd[*] + +#dump 1 all custom 1000 dump_${params}_2d.lammpstrj id type & +# x y z xu yu zu mux muy muz fx fy fz +#dump_modify 1 first yes sort id + +timestep 0.00001 +thermo 100 + +# main run +run 3000 diff --git a/examples/USER/brownian/sphere/in3d.sphere b/examples/USER/brownian/sphere/in3d.sphere new file mode 100644 index 0000000000..d3264ee450 --- /dev/null +++ b/examples/USER/brownian/sphere/in3d.sphere @@ -0,0 +1,45 @@ +##### overdamped dynamics of a sphere (with dipole attached to it) in 3D##### + +variable rng string uniform +variable seed string 198098 +variable temp string 1.0 +variable gamma_t string 5.0 +variable gamma_r string 0.7 +variable params string ${rng}_${temp}_${gamma_r}_${gamma_t} + +units lj +atom_style hybrid dipole sphere +dimension 3 +newton off + +lattice sc 0.4 +region box block -8 8 -8 8 -8 8 +create_box 1 box +create_atoms 1 box +mass * 1.0 +set type * dipole/random ${seed} 1.0 +velocity all create 1.0 1 loop geom + +neighbor 1.0 bin +neigh_modify every 1 delay 1 check yes + +pair_style none + +fix 1 all brownian/sphere ${temp} ${seed} rng ${rng} gamma_r ${gamma_r} gamma_t ${gamma_t} + +#initialisation for the main run + +# MSD +compute msd all msd + +thermo_style custom step ke pe c_msd[*] + +#dump 1 all custom 1000 dump_${params}_3d.lammpstrj id type & +# x y z xu yu zu mux muy muz fx fy fz +#dump_modify 1 first yes sort id + +timestep 0.00001 +thermo 100 + +# main run +run 3000 diff --git a/examples/USER/brownian/sphere/log.11May2021.in2d.sphere.g++.1 b/examples/USER/brownian/sphere/log.11May2021.in2d.sphere.g++.1 new file mode 100644 index 0000000000..917f2b63fb --- /dev/null +++ b/examples/USER/brownian/sphere/log.11May2021.in2d.sphere.g++.1 @@ -0,0 +1,126 @@ +LAMMPS (8 Apr 2021) + using 1 OpenMP thread(s) per MPI task +##### overdamped dynamics of a sphere (with dipole attached to it) in 2D ##### + +variable rng string uniform +variable seed string 198098 +variable temp string 1.0 +variable gamma_t string 5.0 +variable gamma_r string 0.7 +variable params string ${rng}_${temp}_${gamma_r}_${gamma_t} +variable params string uniform_${temp}_${gamma_r}_${gamma_t} +variable params string uniform_1.0_${gamma_r}_${gamma_t} +variable params string uniform_1.0_0.7_${gamma_t} +variable params string uniform_1.0_0.7_5.0 + +units lj +atom_style hybrid dipole sphere +WARNING: Atom style hybrid defines both, per-type and per-atom masses; both must be set, but only per-atom masses will be used (src/atom_vec_hybrid.cpp:156) +dimension 2 +newton off + +lattice sq 0.4 +Lattice spacing in x,y,z = 1.5811388 1.5811388 1.5811388 +region box block -30 30 -30 30 -0.2 0.2 +create_box 1 box +Created orthogonal box = (-47.434165 -47.434165 -0.31622777) to (47.434165 47.434165 0.31622777) + 1 by 1 by 1 MPI processor grid +create_atoms 1 box +Created 3600 atoms + create_atoms CPU = 0.005 seconds +mass * 1.0 +set type * dipole/random ${seed} 1.0 +set type * dipole/random 198098 1.0 +Setting atom values ... + 3600 settings made for dipole/random +velocity all create 1.0 1 loop geom + +neighbor 1.0 bin +neigh_modify every 1 delay 1 check yes + +pair_style none + +fix 1 all brownian/sphere ${temp} ${seed} rng ${rng} gamma_r ${gamma_r} gamma_t ${gamma_t} +fix 1 all brownian/sphere 1.0 ${seed} rng ${rng} gamma_r ${gamma_r} gamma_t ${gamma_t} +fix 1 all brownian/sphere 1.0 198098 rng ${rng} gamma_r ${gamma_r} gamma_t ${gamma_t} +fix 1 all brownian/sphere 1.0 198098 rng uniform gamma_r ${gamma_r} gamma_t ${gamma_t} +fix 1 all brownian/sphere 1.0 198098 rng uniform gamma_r 0.7 gamma_t ${gamma_t} +fix 1 all brownian/sphere 1.0 198098 rng uniform gamma_r 0.7 gamma_t 5.0 + +#initialisation for the main run + +# MSD +compute msd all msd + +thermo_style custom step ke pe c_msd[*] + +#dump 1 all custom 1000 dump_${params}_2d.lammpstrj id type # x y z xu yu zu mux muy muz fx fy fz +#dump_modify 1 first yes sort id + +timestep 0.00001 +thermo 100 + +# main run +run 3000 +WARNING: No pairwise cutoff or binsize set. Atom sorting therefore disabled. (src/atom.cpp:2141) +WARNING: Communication cutoff is 0.0. No ghost atoms will be generated. Atoms may get lost. (src/comm_brick.cpp:167) +Per MPI rank memory allocation (min/avg/max) = 4.664 | 4.664 | 4.664 Mbytes +Step KinEng PotEng c_msd[1] c_msd[2] c_msd[3] c_msd[4] + 0 0.99972222 0 0 0 0 0 + 100 20867.136 0 0.00040006704 0.00039570887 0 0.00079577592 + 200 20835.491 0 0.00077560901 0.00080455484 0 0.0015801638 + 300 20813.122 0 0.0011737739 0.0012172689 0 0.0023910428 + 400 21137.397 0 0.0015587675 0.0016096093 0 0.0031683768 + 500 21167.188 0 0.0019294105 0.0020251322 0 0.0039545428 + 600 21345.908 0 0.0023105313 0.0024111742 0 0.0047217054 + 700 21086.272 0 0.0027236116 0.0027846006 0 0.0055082122 + 800 20840.906 0 0.0031505299 0.0031810732 0 0.0063316031 + 900 20916.456 0 0.0035525852 0.0035981301 0 0.0071507153 + 1000 20752.249 0 0.0039147929 0.0039791172 0 0.0078939101 + 1100 20643.612 0 0.0042977921 0.0043701484 0 0.0086679405 + 1200 21085.63 0 0.0045584242 0.0047475091 0 0.0093059332 + 1300 20900.794 0 0.0049718803 0.0051481706 0 0.010120051 + 1400 20980.731 0 0.0054234603 0.0054230724 0 0.010846533 + 1500 20916.308 0 0.0058502946 0.0058114313 0 0.011661726 + 1600 20949.786 0 0.0062258463 0.006208129 0 0.012433975 + 1700 20531.205 0 0.0066276219 0.006595921 0 0.013223543 + 1800 21418.472 0 0.0070077409 0.007030461 0 0.014038202 + 1900 21291.928 0 0.0074052208 0.0074333041 0 0.014838525 + 2000 20893.895 0 0.0077407477 0.007901402 0 0.01564215 + 2100 21218.001 0 0.0080384756 0.0082611258 0 0.016299601 + 2200 21116.189 0 0.0084325164 0.008617977 0 0.017050493 + 2300 20718.83 0 0.0089455345 0.0091768161 0 0.018122351 + 2400 20719.164 0 0.0093666455 0.0095272546 0 0.0188939 + 2500 20991.382 0 0.009706795 0.0098256506 0 0.019532446 + 2600 20515.74 0 0.0099247069 0.010329841 0 0.020254548 + 2700 21001.55 0 0.010448354 0.010693502 0 0.021141855 + 2800 21363.824 0 0.010990971 0.011142092 0 0.022133063 + 2900 20497.025 0 0.011399704 0.011504868 0 0.022904573 + 3000 20726.572 0 0.011785354 0.01187482 0 0.023660175 +Loop time of 1.76023 on 1 procs for 3000 steps with 3600 atoms + +Performance: 1472.538 tau/day, 1704.326 timesteps/s +99.9% CPU use with 1 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 0 | 0 | 0 | 0.0 | 0.00 +Neigh | 0 | 0 | 0 | 0.0 | 0.00 +Comm | 0.026518 | 0.026518 | 0.026518 | 0.0 | 1.51 +Output | 0.0040107 | 0.0040107 | 0.0040107 | 0.0 | 0.23 +Modify | 1.6194 | 1.6194 | 1.6194 | 0.0 | 92.00 +Other | | 0.1103 | | | 6.27 + +Nlocal: 3600.00 ave 3600 max 3600 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Nghost: 121.000 ave 121 max 121 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Neighs: 0.00000 ave 0 max 0 min +Histogram: 1 0 0 0 0 0 0 0 0 0 + +Total # of neighbors = 0 +Ave neighs/atom = 0.0000000 +Neighbor list builds = 0 +Dangerous builds = 0 +Total wall time: 0:00:01 diff --git a/examples/USER/brownian/sphere/log.11May2021.in2d.sphere.g++.4 b/examples/USER/brownian/sphere/log.11May2021.in2d.sphere.g++.4 new file mode 100644 index 0000000000..95602a809c --- /dev/null +++ b/examples/USER/brownian/sphere/log.11May2021.in2d.sphere.g++.4 @@ -0,0 +1,126 @@ +LAMMPS (8 Apr 2021) + using 1 OpenMP thread(s) per MPI task +##### overdamped dynamics of a sphere (with dipole attached to it) in 2D ##### + +variable rng string uniform +variable seed string 198098 +variable temp string 1.0 +variable gamma_t string 5.0 +variable gamma_r string 0.7 +variable params string ${rng}_${temp}_${gamma_r}_${gamma_t} +variable params string uniform_${temp}_${gamma_r}_${gamma_t} +variable params string uniform_1.0_${gamma_r}_${gamma_t} +variable params string uniform_1.0_0.7_${gamma_t} +variable params string uniform_1.0_0.7_5.0 + +units lj +atom_style hybrid dipole sphere +WARNING: Atom style hybrid defines both, per-type and per-atom masses; both must be set, but only per-atom masses will be used (src/atom_vec_hybrid.cpp:156) +dimension 2 +newton off + +lattice sq 0.4 +Lattice spacing in x,y,z = 1.5811388 1.5811388 1.5811388 +region box block -30 30 -30 30 -0.2 0.2 +create_box 1 box +Created orthogonal box = (-47.434165 -47.434165 -0.31622777) to (47.434165 47.434165 0.31622777) + 2 by 2 by 1 MPI processor grid +create_atoms 1 box +Created 3600 atoms + create_atoms CPU = 0.002 seconds +mass * 1.0 +set type * dipole/random ${seed} 1.0 +set type * dipole/random 198098 1.0 +Setting atom values ... + 3600 settings made for dipole/random +velocity all create 1.0 1 loop geom + +neighbor 1.0 bin +neigh_modify every 1 delay 1 check yes + +pair_style none + +fix 1 all brownian/sphere ${temp} ${seed} rng ${rng} gamma_r ${gamma_r} gamma_t ${gamma_t} +fix 1 all brownian/sphere 1.0 ${seed} rng ${rng} gamma_r ${gamma_r} gamma_t ${gamma_t} +fix 1 all brownian/sphere 1.0 198098 rng ${rng} gamma_r ${gamma_r} gamma_t ${gamma_t} +fix 1 all brownian/sphere 1.0 198098 rng uniform gamma_r ${gamma_r} gamma_t ${gamma_t} +fix 1 all brownian/sphere 1.0 198098 rng uniform gamma_r 0.7 gamma_t ${gamma_t} +fix 1 all brownian/sphere 1.0 198098 rng uniform gamma_r 0.7 gamma_t 5.0 + +#initialisation for the main run + +# MSD +compute msd all msd + +thermo_style custom step ke pe c_msd[*] + +#dump 1 all custom 1000 dump_${params}_2d.lammpstrj id type # x y z xu yu zu mux muy muz fx fy fz +#dump_modify 1 first yes sort id + +timestep 0.00001 +thermo 100 + +# main run +run 3000 +WARNING: No pairwise cutoff or binsize set. Atom sorting therefore disabled. (src/atom.cpp:2141) +WARNING: Communication cutoff is 0.0. No ghost atoms will be generated. Atoms may get lost. (src/comm_brick.cpp:167) +Per MPI rank memory allocation (min/avg/max) = 4.664 | 4.664 | 4.664 Mbytes +Step KinEng PotEng c_msd[1] c_msd[2] c_msd[3] c_msd[4] + 0 0.99972222 0 0 0 0 0 + 100 21085.797 0 0.00042014118 0.00040399828 0 0.00082413946 + 200 20598.717 0 0.00081715618 0.00082613236 0 0.0016432885 + 300 21040.226 0 0.0012412527 0.0012520475 0 0.0024933002 + 400 21289.734 0 0.0016129899 0.001634482 0 0.003247472 + 500 20951.595 0 0.0020104279 0.0020197694 0 0.0040301973 + 600 20984.974 0 0.0023965593 0.0024277086 0 0.0048242679 + 700 21252.602 0 0.0028349303 0.0028407812 0 0.0056757114 + 800 20951.95 0 0.0032674595 0.0032573476 0 0.0065248071 + 900 20828.611 0 0.003647953 0.0036650963 0 0.0073130493 + 1000 21073.256 0 0.0040238604 0.0040103537 0 0.0080342142 + 1100 21104.396 0 0.0043694059 0.0044146515 0 0.0087840574 + 1200 20580.591 0 0.0047638237 0.0047646659 0 0.0095284896 + 1300 20667.623 0 0.0051512568 0.0051134445 0 0.010264701 + 1400 20466.72 0 0.0055921578 0.005517863 0 0.011110021 + 1500 20842.366 0 0.0059747304 0.0059374031 0 0.011912134 + 1600 20867.02 0 0.0065493697 0.0064163066 0 0.012965676 + 1700 21021.077 0 0.0070208005 0.0068164842 0 0.013837285 + 1800 21191.183 0 0.0073708939 0.0073226521 0 0.014693546 + 1900 20792.8 0 0.0076984189 0.0077400043 0 0.015438423 + 2000 21296.326 0 0.0081882545 0.0081503672 0 0.016338622 + 2100 21085.097 0 0.008596146 0.0086041272 0 0.017200273 + 2200 20506.523 0 0.0089905439 0.0091045462 0 0.01809509 + 2300 21068.555 0 0.0094163509 0.0094703314 0 0.018886682 + 2400 21128.867 0 0.0097349212 0.0098535832 0 0.019588504 + 2500 21009.514 0 0.010218059 0.010244621 0 0.020462679 + 2600 21177.52 0 0.01060437 0.010642719 0 0.021247089 + 2700 20832.56 0 0.011052623 0.011078899 0 0.022131522 + 2800 21298.053 0 0.011439708 0.011587702 0 0.023027411 + 2900 21002.28 0 0.011863624 0.01199682 0 0.023860443 + 3000 20737.939 0 0.012229608 0.012324579 0 0.024554186 +Loop time of 0.492798 on 4 procs for 3000 steps with 3600 atoms + +Performance: 5259.763 tau/day, 6087.689 timesteps/s +96.5% CPU use with 4 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 0 | 0 | 0 | 0.0 | 0.00 +Neigh | 0 | 0 | 0 | 0.0 | 0.00 +Comm | 0.018005 | 0.021524 | 0.025207 | 2.2 | 4.37 +Output | 0.0013187 | 0.0018334 | 0.0032332 | 1.9 | 0.37 +Modify | 0.37545 | 0.38775 | 0.40664 | 1.9 | 78.68 +Other | | 0.08169 | | | 16.58 + +Nlocal: 900.000 ave 900 max 900 min +Histogram: 4 0 0 0 0 0 0 0 0 0 +Nghost: 61.0000 ave 61 max 61 min +Histogram: 4 0 0 0 0 0 0 0 0 0 +Neighs: 0.00000 ave 0 max 0 min +Histogram: 4 0 0 0 0 0 0 0 0 0 + +Total # of neighbors = 0 +Ave neighs/atom = 0.0000000 +Neighbor list builds = 0 +Dangerous builds = 0 +Total wall time: 0:00:00 diff --git a/examples/USER/brownian/sphere/log.11May2021.in3d.sphere.g++.1 b/examples/USER/brownian/sphere/log.11May2021.in3d.sphere.g++.1 new file mode 100644 index 0000000000..20048f96f1 --- /dev/null +++ b/examples/USER/brownian/sphere/log.11May2021.in3d.sphere.g++.1 @@ -0,0 +1,126 @@ +LAMMPS (8 Apr 2021) + using 1 OpenMP thread(s) per MPI task +##### overdamped dynamics of a sphere (with dipole attached to it) in 3D##### + +variable rng string uniform +variable seed string 198098 +variable temp string 1.0 +variable gamma_t string 5.0 +variable gamma_r string 0.7 +variable params string ${rng}_${temp}_${gamma_r}_${gamma_t} +variable params string uniform_${temp}_${gamma_r}_${gamma_t} +variable params string uniform_1.0_${gamma_r}_${gamma_t} +variable params string uniform_1.0_0.7_${gamma_t} +variable params string uniform_1.0_0.7_5.0 + +units lj +atom_style hybrid dipole sphere +WARNING: Atom style hybrid defines both, per-type and per-atom masses; both must be set, but only per-atom masses will be used (src/atom_vec_hybrid.cpp:156) +dimension 3 +newton off + +lattice sc 0.4 +Lattice spacing in x,y,z = 1.3572088 1.3572088 1.3572088 +region box block -8 8 -8 8 -8 8 +create_box 1 box +Created orthogonal box = (-10.857670 -10.857670 -10.857670) to (10.857670 10.857670 10.857670) + 1 by 1 by 1 MPI processor grid +create_atoms 1 box +Created 4096 atoms + create_atoms CPU = 0.005 seconds +mass * 1.0 +set type * dipole/random ${seed} 1.0 +set type * dipole/random 198098 1.0 +Setting atom values ... + 4096 settings made for dipole/random +velocity all create 1.0 1 loop geom + +neighbor 1.0 bin +neigh_modify every 1 delay 1 check yes + +pair_style none + +fix 1 all brownian/sphere ${temp} ${seed} rng ${rng} gamma_r ${gamma_r} gamma_t ${gamma_t} +fix 1 all brownian/sphere 1.0 ${seed} rng ${rng} gamma_r ${gamma_r} gamma_t ${gamma_t} +fix 1 all brownian/sphere 1.0 198098 rng ${rng} gamma_r ${gamma_r} gamma_t ${gamma_t} +fix 1 all brownian/sphere 1.0 198098 rng uniform gamma_r ${gamma_r} gamma_t ${gamma_t} +fix 1 all brownian/sphere 1.0 198098 rng uniform gamma_r 0.7 gamma_t ${gamma_t} +fix 1 all brownian/sphere 1.0 198098 rng uniform gamma_r 0.7 gamma_t 5.0 + +#initialisation for the main run + +# MSD +compute msd all msd + +thermo_style custom step ke pe c_msd[*] + +#dump 1 all custom 1000 dump_${params}_3d.lammpstrj id type # x y z xu yu zu mux muy muz fx fy fz +#dump_modify 1 first yes sort id + +timestep 0.00001 +thermo 100 + +# main run +run 3000 +WARNING: No pairwise cutoff or binsize set. Atom sorting therefore disabled. (src/atom.cpp:2141) +WARNING: Communication cutoff is 0.0. No ghost atoms will be generated. Atoms may get lost. (src/comm_brick.cpp:167) +Per MPI rank memory allocation (min/avg/max) = 4.737 | 4.737 | 4.737 Mbytes +Step KinEng PotEng c_msd[1] c_msd[2] c_msd[3] c_msd[4] + 0 1.4996338 0 0 0 0 0 + 100 32032.279 0 0.00040227568 0.00039159837 0.00040147813 0.0011953522 + 200 31673.093 0 0.00077657885 0.00077292327 0.00079963705 0.0023491392 + 300 31476.164 0 0.0011712083 0.0011606723 0.0012089537 0.0035408343 + 400 31911.374 0 0.0015803424 0.001562091 0.0016042321 0.0047466655 + 500 31182.011 0 0.0019677217 0.0019269105 0.0020015977 0.0058962298 + 600 31206.05 0 0.0023360975 0.0023134398 0.0024213372 0.0070708745 + 700 31278.057 0 0.0026966955 0.0027129858 0.0028721373 0.0082818187 + 800 31677.724 0 0.0031197964 0.003134834 0.0032726303 0.0095272607 + 900 31312.741 0 0.0035636612 0.0035573653 0.0037328373 0.010853864 + 1000 31426.075 0 0.0039774626 0.003952159 0.0041879386 0.01211756 + 1100 31361.699 0 0.0044256852 0.004320566 0.004638132 0.013384383 + 1200 31559.778 0 0.0048338539 0.0047210601 0.0050296056 0.01458452 + 1300 31716.797 0 0.0052239651 0.0050796723 0.0054794684 0.015783106 + 1400 31231.077 0 0.0055890568 0.005472377 0.0059264123 0.016987846 + 1500 31605.513 0 0.0059876582 0.0058974054 0.0063452478 0.018230311 + 1600 31551.402 0 0.006413094 0.0062665632 0.0067442106 0.019423868 + 1700 31725.868 0 0.0068244611 0.0067189707 0.0071424779 0.02068591 + 1800 31385.794 0 0.0071570297 0.0070502303 0.0075240296 0.02173129 + 1900 31754.094 0 0.0075638662 0.0074243015 0.0079935325 0.0229817 + 2000 31668.959 0 0.0080059944 0.0079019753 0.0084000614 0.024308031 + 2100 31781.994 0 0.0084108141 0.0082719077 0.0088004977 0.02548322 + 2200 31455.021 0 0.0088844434 0.0086931769 0.0091916929 0.026769313 + 2300 31273.079 0 0.0093155639 0.0091027782 0.0095364621 0.027954804 + 2400 31283.781 0 0.0098441686 0.0094496218 0.0099279073 0.029221698 + 2500 31758.315 0 0.010372129 0.0097843406 0.010334653 0.030491123 + 2600 31780.442 0 0.010770862 0.010313119 0.010637545 0.031721525 + 2700 31552.277 0 0.011268703 0.010693437 0.01110762 0.033069759 + 2800 31124.693 0 0.011661333 0.011100115 0.011480624 0.034242072 + 2900 31438.795 0 0.012068847 0.011346633 0.011842006 0.035257486 + 3000 31574.258 0 0.012482632 0.011691477 0.012210207 0.036384317 +Loop time of 2.8531 on 1 procs for 3000 steps with 4096 atoms + +Performance: 908.486 tau/day, 1051.488 timesteps/s +99.9% CPU use with 1 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 0 | 0 | 0 | 0.0 | 0.00 +Neigh | 3.2425e-05 | 3.2425e-05 | 3.2425e-05 | 0.0 | 0.00 +Comm | 0.13219 | 0.13219 | 0.13219 | 0.0 | 4.63 +Output | 0.0045686 | 0.0045686 | 0.0045686 | 0.0 | 0.16 +Modify | 2.5857 | 2.5857 | 2.5857 | 0.0 | 90.63 +Other | | 0.1307 | | | 4.58 + +Nlocal: 4096.00 ave 4096 max 4096 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Nghost: 0.00000 ave 0 max 0 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Neighs: 0.00000 ave 0 max 0 min +Histogram: 1 0 0 0 0 0 0 0 0 0 + +Total # of neighbors = 0 +Ave neighs/atom = 0.0000000 +Neighbor list builds = 1 +Dangerous builds = 0 +Total wall time: 0:00:02 diff --git a/examples/USER/brownian/sphere/log.11May2021.in3d.sphere.g++.4 b/examples/USER/brownian/sphere/log.11May2021.in3d.sphere.g++.4 new file mode 100644 index 0000000000..57dc6d5f11 --- /dev/null +++ b/examples/USER/brownian/sphere/log.11May2021.in3d.sphere.g++.4 @@ -0,0 +1,126 @@ +LAMMPS (8 Apr 2021) + using 1 OpenMP thread(s) per MPI task +##### overdamped dynamics of a sphere (with dipole attached to it) in 3D##### + +variable rng string uniform +variable seed string 198098 +variable temp string 1.0 +variable gamma_t string 5.0 +variable gamma_r string 0.7 +variable params string ${rng}_${temp}_${gamma_r}_${gamma_t} +variable params string uniform_${temp}_${gamma_r}_${gamma_t} +variable params string uniform_1.0_${gamma_r}_${gamma_t} +variable params string uniform_1.0_0.7_${gamma_t} +variable params string uniform_1.0_0.7_5.0 + +units lj +atom_style hybrid dipole sphere +WARNING: Atom style hybrid defines both, per-type and per-atom masses; both must be set, but only per-atom masses will be used (src/atom_vec_hybrid.cpp:156) +dimension 3 +newton off + +lattice sc 0.4 +Lattice spacing in x,y,z = 1.3572088 1.3572088 1.3572088 +region box block -8 8 -8 8 -8 8 +create_box 1 box +Created orthogonal box = (-10.857670 -10.857670 -10.857670) to (10.857670 10.857670 10.857670) + 2 by 1 by 2 MPI processor grid +create_atoms 1 box +Created 4096 atoms + create_atoms CPU = 0.006 seconds +mass * 1.0 +set type * dipole/random ${seed} 1.0 +set type * dipole/random 198098 1.0 +Setting atom values ... + 4096 settings made for dipole/random +velocity all create 1.0 1 loop geom + +neighbor 1.0 bin +neigh_modify every 1 delay 1 check yes + +pair_style none + +fix 1 all brownian/sphere ${temp} ${seed} rng ${rng} gamma_r ${gamma_r} gamma_t ${gamma_t} +fix 1 all brownian/sphere 1.0 ${seed} rng ${rng} gamma_r ${gamma_r} gamma_t ${gamma_t} +fix 1 all brownian/sphere 1.0 198098 rng ${rng} gamma_r ${gamma_r} gamma_t ${gamma_t} +fix 1 all brownian/sphere 1.0 198098 rng uniform gamma_r ${gamma_r} gamma_t ${gamma_t} +fix 1 all brownian/sphere 1.0 198098 rng uniform gamma_r 0.7 gamma_t ${gamma_t} +fix 1 all brownian/sphere 1.0 198098 rng uniform gamma_r 0.7 gamma_t 5.0 + +#initialisation for the main run + +# MSD +compute msd all msd + +thermo_style custom step ke pe c_msd[*] + +#dump 1 all custom 1000 dump_${params}_3d.lammpstrj id type # x y z xu yu zu mux muy muz fx fy fz +#dump_modify 1 first yes sort id + +timestep 0.00001 +thermo 100 + +# main run +run 3000 +WARNING: No pairwise cutoff or binsize set. Atom sorting therefore disabled. (src/atom.cpp:2141) +WARNING: Communication cutoff is 0.0. No ghost atoms will be generated. Atoms may get lost. (src/comm_brick.cpp:167) +Per MPI rank memory allocation (min/avg/max) = 4.694 | 4.694 | 4.694 Mbytes +Step KinEng PotEng c_msd[1] c_msd[2] c_msd[3] c_msd[4] + 0 1.4996338 0 0 0 0 0 + 100 30882.707 0 0.00040787161 0.00039391576 0.00040796913 0.0012097565 + 200 31370.751 0 0.00081742036 0.00078240455 0.0008195167 0.0024193416 + 300 31469.51 0 0.0012118299 0.0011819412 0.0012477119 0.003641483 + 400 31696.58 0 0.0015540547 0.0015849689 0.0015836091 0.0047226327 + 500 31488.269 0 0.0019638041 0.0019659637 0.0020547832 0.005984551 + 600 30942.589 0 0.0023273784 0.0023572171 0.0024715245 0.00715612 + 700 31228.473 0 0.0027821732 0.002735338 0.0028734675 0.0083909787 + 800 31426.92 0 0.0031663838 0.0031092782 0.0033231014 0.0095987634 + 900 31447.595 0 0.003539588 0.003564381 0.003753036 0.010857005 + 1000 31363.911 0 0.0039854308 0.003937555 0.0041203919 0.012043378 + 1100 31522.958 0 0.0043009285 0.0043676491 0.0044799414 0.013148519 + 1200 31403.033 0 0.0046361199 0.0047513598 0.0049014974 0.014288977 + 1300 31752.182 0 0.0049824718 0.0051327113 0.0053130614 0.015428244 + 1400 31336.955 0 0.0054251445 0.0055442325 0.0057472998 0.016716677 + 1500 31224.306 0 0.0059295596 0.0059920697 0.0061375228 0.018059152 + 1600 31744.535 0 0.0063845142 0.0063600989 0.0064833215 0.019227935 + 1700 31472.081 0 0.0068360092 0.0067985824 0.0069464303 0.020581022 + 1800 31577.334 0 0.0073001079 0.0071355564 0.0073400543 0.021775719 + 1900 31521.234 0 0.0077178677 0.0074371106 0.007708008 0.022862986 + 2000 31045.148 0 0.0080515968 0.0078583776 0.0081000219 0.024009996 + 2100 31289.809 0 0.0084280175 0.0082322226 0.0084475904 0.02510783 + 2200 31505.455 0 0.008802925 0.0085708943 0.0087648194 0.026138639 + 2300 31882.722 0 0.0092223105 0.0089242925 0.0092643028 0.027410906 + 2400 31028.15 0 0.0095737559 0.0093585981 0.0096771837 0.028609538 + 2500 31581.041 0 0.0099316284 0.009785264 0.010100235 0.029817127 + 2600 31272.119 0 0.010332986 0.01007291 0.010474606 0.030880502 + 2700 31537.8 0 0.010751592 0.010565273 0.01093107 0.032247935 + 2800 31060.697 0 0.011156729 0.011010751 0.011260025 0.033427506 + 2900 31541.612 0 0.011542003 0.011499419 0.011642873 0.034684295 + 3000 31305.382 0 0.011876832 0.011866445 0.012052577 0.035795854 +Loop time of 1.00142 on 4 procs for 3000 steps with 4096 atoms + +Performance: 2588.329 tau/day, 2995.751 timesteps/s +95.2% CPU use with 4 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 0 | 0 | 0 | 0.0 | 0.00 +Neigh | 0 | 0 | 0 | 0.0 | 0.00 +Comm | 0.084416 | 0.10685 | 0.12695 | 5.7 | 10.67 +Output | 0.001471 | 0.0019466 | 0.0033245 | 1.8 | 0.19 +Modify | 0.66585 | 0.70645 | 0.78995 | 6.0 | 70.54 +Other | | 0.1862 | | | 18.59 + +Nlocal: 1024.00 ave 1024 max 1024 min +Histogram: 4 0 0 0 0 0 0 0 0 0 +Nghost: 353.000 ave 353 max 353 min +Histogram: 4 0 0 0 0 0 0 0 0 0 +Neighs: 0.00000 ave 0 max 0 min +Histogram: 4 0 0 0 0 0 0 0 0 0 + +Total # of neighbors = 0 +Ave neighs/atom = 0.0000000 +Neighbor list builds = 0 +Dangerous builds = 0 +Total wall time: 0:00:01 diff --git a/examples/USER/brownian/spherical_ABP/in2d.abp b/examples/USER/brownian/spherical_ABP/in2d.abp new file mode 100644 index 0000000000..bd7dce13b6 --- /dev/null +++ b/examples/USER/brownian/spherical_ABP/in2d.abp @@ -0,0 +1,60 @@ +# 2D overdamped active brownian particle dynamics (ABP) +# with WCA potential + +variable gamma_t string 1.0 +variable gamma_r string 1.0 +variable temp string 1.0 +variable seed equal 1974019 +variable fp string 4.0 +variable params string ${temp}_${gamma_t}_${gamma_r}_${fp} + +units lj +atom_style hybrid dipole sphere +dimension 2 +newton off + +lattice sq 0.4 +region box block -16 16 -16 16 -0.2 0.2 +create_box 1 box +create_atoms 1 box +mass * 1.0 +set type * dipole/random ${seed} 1.0 +velocity all create 1.0 1 loop geom + +# more careful with neighbors since higher diffusion in abps +neighbor 1.0 bin +neigh_modify every 1 delay 1 check yes + +# WCA potential (purely repulsive) +pair_style lj/cut 2.5 +pair_coeff * * 1.0 1.0 1.1224 +pair_modify shift yes + +# overdamped brownian dynamics time-step +fix step all brownian/sphere ${temp} ${seed} gamma_t ${gamma_t} gamma_r ${gamma_r} +# self-propulsion force along the dipole direction +fix activity all propel/self dipole ${fp} +fix 2 all enforce2d + +compute press all pressure NULL virial + +thermo_style custom step pe ke c_press + +#equilibration +timestep 0.0000000001 +thermo 100 +run 5000 +reset_timestep 0 + + +# MSD +compute msd all msd + +thermo_style custom step temp epair c_msd[*] c_press + + +timestep 0.00001 +thermo 1000 + +# main run +run 20000 diff --git a/examples/USER/brownian/spherical_ABP/in3d.ideal_abp b/examples/USER/brownian/spherical_ABP/in3d.ideal_abp new file mode 100644 index 0000000000..6a6575b510 --- /dev/null +++ b/examples/USER/brownian/spherical_ABP/in3d.ideal_abp @@ -0,0 +1,67 @@ +# 3D overdamped active brownian dynamics with no interactions + +variable gamma_t string 3.0 +variable gamma_r string 1.0 +variable temp string 1.0 +variable seed equal 1974019 +variable fp string 4.0 +variable params string ${temp}_${gamma_t}_${gamma_r}_${fp} + +units lj +atom_style hybrid dipole sphere +dimension 3 +newton off + +lattice sc 0.4 +region box block -8 8 -8 8 -8 8 +create_box 1 box +create_atoms 1 box +mass * 1.0 +set type * dipole/random ${seed} 1.0 +velocity all create 1.0 1 loop geom + +pair_style none + +# overdamped brownian dynamics time-step +fix step all brownian/sphere ${temp} ${seed} gamma_t ${gamma_t} gamma_r ${gamma_r} +# self-propulsion force along the dipole direction +fix activity all propel/self dipole ${fp} + +compute press all pressure NULL virial + +thermo_style custom step ke pe c_press + +#equilibration +timestep 0.0000000001 +thermo 100 +run 5000 +reset_timestep 0 + +# MSD to demonstrate expected diffusive behaviour for ideal active +# brownian motion, which is +# +# MSD = (2*d*kb*T/gamma_t + 2*fp**2*gamma_r/(kb*T*gamma_t**2*(d-1)))*t +# + 2*fp**2*gamma_r**2/(gamma_t**2*(d-1)**2*(kb*T)**2)*(e^(-(d-1)*t*kb*T/gamma_r)-1) +# +# with d being simulation dimension +compute msd all msd + +thermo_style custom step ke pe c_msd[*] c_press + + +timestep 0.00001 +thermo 1000 + +# main run +run 12000 + +# if you want to check that rotational diffusion is behaving as expected, +# uncomment next three lines for dump output and then plot , +# which should decay exponentially with timescale (d-1)*D_r (with d +# being simulation dimension) + +#dump 1 all custom 2000 dump_ideal_${params}_3d.lammpstrj id type & +# x y xu yu mux muy muz fx fy fz +#dump_modify 1 first yes sort id + +#run 120000 diff --git a/examples/USER/brownian/spherical_ABP/log.11May2021.in2d.apb.g++.1 b/examples/USER/brownian/spherical_ABP/log.11May2021.in2d.apb.g++.1 new file mode 100644 index 0000000000..452d8a3d52 --- /dev/null +++ b/examples/USER/brownian/spherical_ABP/log.11May2021.in2d.apb.g++.1 @@ -0,0 +1,221 @@ +LAMMPS (8 Apr 2021) + using 1 OpenMP thread(s) per MPI task +# 2D overdamped active brownian particle dynamics (ABP) +# with WCA potential + +variable gamma_t string 1.0 +variable gamma_r string 1.0 +variable temp string 1.0 +variable seed equal 1974019 +variable fp string 4.0 +variable params string ${temp}_${gamma_t}_${gamma_r}_${fp} +variable params string 1.0_${gamma_t}_${gamma_r}_${fp} +variable params string 1.0_1.0_${gamma_r}_${fp} +variable params string 1.0_1.0_1.0_${fp} +variable params string 1.0_1.0_1.0_4.0 + +units lj +atom_style hybrid dipole sphere +WARNING: Atom style hybrid defines both, per-type and per-atom masses; both must be set, but only per-atom masses will be used (src/atom_vec_hybrid.cpp:156) +dimension 2 +newton off + +lattice sq 0.4 +Lattice spacing in x,y,z = 1.5811388 1.5811388 1.5811388 +region box block -16 16 -16 16 -0.2 0.2 +create_box 1 box +Created orthogonal box = (-25.298221 -25.298221 -0.31622777) to (25.298221 25.298221 0.31622777) + 1 by 1 by 1 MPI processor grid +create_atoms 1 box +Created 1024 atoms + create_atoms CPU = 0.002 seconds +mass * 1.0 +set type * dipole/random ${seed} 1.0 +set type * dipole/random 1974019 1.0 +Setting atom values ... + 1024 settings made for dipole/random +velocity all create 1.0 1 loop geom + +# more careful with neighbors since higher diffusion in abps +neighbor 1.0 bin +neigh_modify every 1 delay 1 check yes + +# WCA potential (purely repulsive) +pair_style lj/cut 2.5 +pair_coeff * * 1.0 1.0 1.1224 +pair_modify shift yes + +# overdamped brownian dynamics time-step +fix step all brownian/sphere ${temp} ${seed} gamma_t ${gamma_t} gamma_r ${gamma_r} +fix step all brownian/sphere 1.0 ${seed} gamma_t ${gamma_t} gamma_r ${gamma_r} +fix step all brownian/sphere 1.0 1974019 gamma_t ${gamma_t} gamma_r ${gamma_r} +fix step all brownian/sphere 1.0 1974019 gamma_t 1.0 gamma_r ${gamma_r} +fix step all brownian/sphere 1.0 1974019 gamma_t 1.0 gamma_r 1.0 +# self-propulsion force along the dipole direction +fix activity all propel/self dipole ${fp} +fix activity all propel/self dipole 4.0 +fix 2 all enforce2d + +compute press all pressure NULL virial + +thermo_style custom step pe ke c_press + +#equilibration +timestep 0.0000000001 +thermo 100 +run 5000 +Neighbor list info ... + update every 1 steps, delay 1 steps, check yes + max neighbors/atom: 2000, page size: 100000 + master list distance cutoff = 2.1224 + ghost atom cutoff = 2.1224 + binsize = 1.0612, bins = 48 48 1 + 1 neighbor lists, perpetual/occasional/extra = 1 0 0 + (1) pair lj/cut, perpetual + attributes: half, newton off + pair build: half/bin/newtoff + stencil: half/bin/2d/newtoff + bin: standard +Per MPI rank memory allocation (min/avg/max) = 5.066 | 5.066 | 5.066 Mbytes +Step PotEng KinEng c_press + 0 0 0.99902344 -0.53979198 + 100 0 1.026585e+10 -0.5398101 + 200 0 1.0630628e+10 -0.53977393 + 300 0 1.03483e+10 -0.53977041 + 400 0 1.049279e+10 -0.53974314 + 500 0 1.0832067e+10 -0.53979451 + 600 0 1.0403632e+10 -0.53976233 + 700 0 1.0334726e+10 -0.53976174 + 800 0 1.0119596e+10 -0.53969338 + 900 0 1.0786136e+10 -0.53970415 + 1000 0 1.0539036e+10 -0.53974577 + 1100 0 1.0643695e+10 -0.53982431 + 1200 0 1.0234642e+10 -0.53976823 + 1300 0 1.036268e+10 -0.53981454 + 1400 0 1.0605702e+10 -0.53988117 + 1500 0 1.0517916e+10 -0.53989207 + 1600 0 1.0564482e+10 -0.53993016 + 1700 0 1.0460152e+10 -0.53984454 + 1800 0 1.0468566e+10 -0.53985574 + 1900 0 1.0474075e+10 -0.53985439 + 2000 0 1.0683568e+10 -0.53987349 + 2100 0 1.0269077e+10 -0.53990709 + 2200 0 1.0386943e+10 -0.53990068 + 2300 0 1.0406078e+10 -0.53978402 + 2400 0 1.0482072e+10 -0.53980757 + 2500 0 1.0442975e+10 -0.53982657 + 2600 0 1.0292103e+10 -0.53985533 + 2700 0 1.1106453e+10 -0.53991861 + 2800 0 1.0395289e+10 -0.53990138 + 2900 0 1.034021e+10 -0.53992375 + 3000 0 1.0434718e+10 -0.53995566 + 3100 0 1.0194094e+10 -0.53993997 + 3200 0 1.0411552e+10 -0.54000097 + 3300 0 1.0214175e+10 -0.53999884 + 3400 0 1.0434719e+10 -0.54000005 + 3500 0 1.0529638e+10 -0.53998281 + 3600 0 1.0406541e+10 -0.54000141 + 3700 0 1.0577151e+10 -0.54002354 + 3800 0 1.0488249e+10 -0.53996003 + 3900 0 1.0316153e+10 -0.54002024 + 4000 0 1.0491289e+10 -0.5400259 + 4100 0 1.0587981e+10 -0.5399811 + 4200 0 1.0332035e+10 -0.53997951 + 4300 0 1.0776469e+10 -0.53994151 + 4400 0 1.0982142e+10 -0.53983842 + 4500 0 1.0796919e+10 -0.5398414 + 4600 0 1.0324249e+10 -0.53979712 + 4700 0 1.0420899e+10 -0.53981967 + 4800 0 1.0274188e+10 -0.53976759 + 4900 0 1.0411535e+10 -0.5397757 + 5000 0 1.0399215e+10 -0.53980199 +Loop time of 1.34285 on 1 procs for 5000 steps with 1024 atoms + +Performance: 0.032 tau/day, 3723.422 timesteps/s +99.8% CPU use with 1 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 0.25309 | 0.25309 | 0.25309 | 0.0 | 18.85 +Neigh | 0 | 0 | 0 | 0.0 | 0.00 +Comm | 0.068734 | 0.068734 | 0.068734 | 0.0 | 5.12 +Output | 0.0012887 | 0.0012887 | 0.0012887 | 0.0 | 0.10 +Modify | 0.96552 | 0.96552 | 0.96552 | 0.0 | 71.90 +Other | | 0.05422 | | | 4.04 + +Nlocal: 1024.00 ave 1024 max 1024 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Nghost: 201.000 ave 201 max 201 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Neighs: 2112.00 ave 2112 max 2112 min +Histogram: 1 0 0 0 0 0 0 0 0 0 + +Total # of neighbors = 2112 +Ave neighs/atom = 2.0625000 +Neighbor list builds = 0 +Dangerous builds = 0 +reset_timestep 0 + + +# MSD +compute msd all msd + +thermo_style custom step temp epair c_msd[*] c_press + + +timestep 0.00001 +thermo 1000 + +# main run +run 20000 +Per MPI rank memory allocation (min/avg/max) = 5.441 | 5.441 | 5.441 Mbytes +Step Temp E_pair c_msd[1] c_msd[2] c_msd[3] c_msd[4] c_press + 0 1.0409381e+10 0 0 0 0 0 -0.53980199 + 1000 107022.73 0.0080050427 0.020451432 0.021388798 0 0.04184023 -0.54900967 + 2000 107475.82 0.017262846 0.040669645 0.044251149 0 0.084920794 -0.42740968 + 3000 105388.35 0.042257875 0.062828995 0.05845782 0 0.12128682 -0.31792184 + 4000 106238.38 0.052733384 0.079036841 0.079396453 0 0.15843329 -0.24243699 + 5000 102904.54 0.088524456 0.095977642 0.099533961 0 0.1955116 -0.093468615 + 6000 105274.15 0.065334999 0.11591691 0.11675531 0 0.23267222 -0.21904478 + 7000 108903.41 0.06724271 0.13694218 0.13914947 0 0.27609164 -0.15913012 + 8000 101451.44 0.097201152 0.15704893 0.16178845 0 0.31883738 -0.055786965 + 9000 106808.72 0.084301668 0.18029391 0.175753 0 0.3560469 0.014898739 + 10000 107381.19 0.088583354 0.2000753 0.19569789 0 0.39577319 0.19417596 + 11000 102105.78 0.081066654 0.22042599 0.21914042 0 0.43956641 0.060574143 + 12000 105384.94 0.098716908 0.24382064 0.24673594 0 0.49055657 0.17067875 + 13000 107479.53 0.099989043 0.26942088 0.27207566 0 0.54149654 0.25514896 + 14000 102938.12 0.093252916 0.28529564 0.28698837 0 0.57228401 0.19976355 + 15000 104408.02 0.11900926 0.31291315 0.31195058 0 0.62486373 0.36956014 + 16000 103447.68 0.09627777 0.34145225 0.33159885 0 0.6730511 0.29857404 + 17000 108400.05 0.11433561 0.36561966 0.36068301 0 0.72630267 0.41922801 + 18000 103363.68 0.11040153 0.38709746 0.39228677 0 0.77938423 0.38111686 + 19000 103310.43 0.10660536 0.41406235 0.40975085 0 0.8238132 0.36022184 + 20000 102692.1 0.13517651 0.43870812 0.44138776 0 0.88009588 0.51144366 +Loop time of 5.66207 on 1 procs for 20000 steps with 1024 atoms + +Performance: 3051.889 tau/day, 3532.279 timesteps/s +99.9% CPU use with 1 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 1.3123 | 1.3123 | 1.3123 | 0.0 | 23.18 +Neigh | 0.011856 | 0.011856 | 0.011856 | 0.0 | 0.21 +Comm | 0.2747 | 0.2747 | 0.2747 | 0.0 | 4.85 +Output | 0.0011516 | 0.0011516 | 0.0011516 | 0.0 | 0.02 +Modify | 3.8451 | 3.8451 | 3.8451 | 0.0 | 67.91 +Other | | 0.2169 | | | 3.83 + +Nlocal: 1024.00 ave 1024 max 1024 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Nghost: 184.000 ave 184 max 184 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Neighs: 2558.00 ave 2558 max 2558 min +Histogram: 1 0 0 0 0 0 0 0 0 0 + +Total # of neighbors = 2558 +Ave neighs/atom = 2.4980469 +Neighbor list builds = 23 +Dangerous builds = 0 +Total wall time: 0:00:07 diff --git a/examples/USER/brownian/spherical_ABP/log.11May2021.in2d.apb.g++.4 b/examples/USER/brownian/spherical_ABP/log.11May2021.in2d.apb.g++.4 new file mode 100644 index 0000000000..aa902fbe80 --- /dev/null +++ b/examples/USER/brownian/spherical_ABP/log.11May2021.in2d.apb.g++.4 @@ -0,0 +1,221 @@ +LAMMPS (8 Apr 2021) + using 1 OpenMP thread(s) per MPI task +# 2D overdamped active brownian particle dynamics (ABP) +# with WCA potential + +variable gamma_t string 1.0 +variable gamma_r string 1.0 +variable temp string 1.0 +variable seed equal 1974019 +variable fp string 4.0 +variable params string ${temp}_${gamma_t}_${gamma_r}_${fp} +variable params string 1.0_${gamma_t}_${gamma_r}_${fp} +variable params string 1.0_1.0_${gamma_r}_${fp} +variable params string 1.0_1.0_1.0_${fp} +variable params string 1.0_1.0_1.0_4.0 + +units lj +atom_style hybrid dipole sphere +WARNING: Atom style hybrid defines both, per-type and per-atom masses; both must be set, but only per-atom masses will be used (src/atom_vec_hybrid.cpp:156) +dimension 2 +newton off + +lattice sq 0.4 +Lattice spacing in x,y,z = 1.5811388 1.5811388 1.5811388 +region box block -16 16 -16 16 -0.2 0.2 +create_box 1 box +Created orthogonal box = (-25.298221 -25.298221 -0.31622777) to (25.298221 25.298221 0.31622777) + 2 by 2 by 1 MPI processor grid +create_atoms 1 box +Created 1024 atoms + create_atoms CPU = 0.001 seconds +mass * 1.0 +set type * dipole/random ${seed} 1.0 +set type * dipole/random 1974019 1.0 +Setting atom values ... + 1024 settings made for dipole/random +velocity all create 1.0 1 loop geom + +# more careful with neighbors since higher diffusion in abps +neighbor 1.0 bin +neigh_modify every 1 delay 1 check yes + +# WCA potential (purely repulsive) +pair_style lj/cut 2.5 +pair_coeff * * 1.0 1.0 1.1224 +pair_modify shift yes + +# overdamped brownian dynamics time-step +fix step all brownian/sphere ${temp} ${seed} gamma_t ${gamma_t} gamma_r ${gamma_r} +fix step all brownian/sphere 1.0 ${seed} gamma_t ${gamma_t} gamma_r ${gamma_r} +fix step all brownian/sphere 1.0 1974019 gamma_t ${gamma_t} gamma_r ${gamma_r} +fix step all brownian/sphere 1.0 1974019 gamma_t 1.0 gamma_r ${gamma_r} +fix step all brownian/sphere 1.0 1974019 gamma_t 1.0 gamma_r 1.0 +# self-propulsion force along the dipole direction +fix activity all propel/self dipole ${fp} +fix activity all propel/self dipole 4.0 +fix 2 all enforce2d + +compute press all pressure NULL virial + +thermo_style custom step pe ke c_press + +#equilibration +timestep 0.0000000001 +thermo 100 +run 5000 +Neighbor list info ... + update every 1 steps, delay 1 steps, check yes + max neighbors/atom: 2000, page size: 100000 + master list distance cutoff = 2.1224 + ghost atom cutoff = 2.1224 + binsize = 1.0612, bins = 48 48 1 + 1 neighbor lists, perpetual/occasional/extra = 1 0 0 + (1) pair lj/cut, perpetual + attributes: half, newton off + pair build: half/bin/newtoff + stencil: half/bin/2d/newtoff + bin: standard +Per MPI rank memory allocation (min/avg/max) = 5.052 | 5.052 | 5.052 Mbytes +Step PotEng KinEng c_press + 0 0 0.99902344 -0.53979198 + 100 0 1.0503521e+10 -0.53983092 + 200 0 1.0390343e+10 -0.5398287 + 300 0 1.0493441e+10 -0.53979247 + 400 0 1.0545991e+10 -0.53978678 + 500 0 1.0266398e+10 -0.53986297 + 600 0 1.0484775e+10 -0.53978746 + 700 0 1.0583596e+10 -0.53969114 + 800 0 1.0521919e+10 -0.53968754 + 900 0 1.0492019e+10 -0.53958643 + 1000 0 1.0298052e+10 -0.53949872 + 1100 0 1.0531424e+10 -0.53955431 + 1200 0 1.0635635e+10 -0.53960048 + 1300 0 1.0633405e+10 -0.53966331 + 1400 0 1.0195401e+10 -0.53968849 + 1500 0 1.0593758e+10 -0.53969763 + 1600 0 1.0425238e+10 -0.53971936 + 1700 0 1.0470017e+10 -0.53981957 + 1800 0 1.0545953e+10 -0.53987747 + 1900 0 1.0425015e+10 -0.53990412 + 2000 0 1.0655092e+10 -0.5399511 + 2100 0 1.0197224e+10 -0.53988687 + 2200 0 1.0448012e+10 -0.53986066 + 2300 0 1.0355268e+10 -0.53980415 + 2400 0 1.0246979e+10 -0.53979737 + 2500 0 1.0021539e+10 -0.5397919 + 2600 0 1.0200824e+10 -0.5397575 + 2700 0 1.0721591e+10 -0.53973512 + 2800 0 1.0354562e+10 -0.5397127 + 2900 0 1.0306795e+10 -0.5396946 + 3000 0 1.0301339e+10 -0.53968642 + 3100 0 1.0435826e+10 -0.53970945 + 3200 0 1.019524e+10 -0.53969746 + 3300 0 1.0550481e+10 -0.53967977 + 3400 0 1.0283446e+10 -0.53971102 + 3500 0 1.0956695e+10 -0.53976173 + 3600 0 1.0271033e+10 -0.53983632 + 3700 0 1.0389461e+10 -0.53977293 + 3800 0 1.0680515e+10 -0.53977425 + 3900 0 1.0072183e+10 -0.53982922 + 4000 0 1.0458036e+10 -0.53980042 + 4100 0 1.0588689e+10 -0.53971405 + 4200 0 1.0068308e+10 -0.5398033 + 4300 0 1.0502064e+10 -0.53981291 + 4400 0 1.0590544e+10 -0.5398346 + 4500 0 1.0411612e+10 -0.5397916 + 4600 0 1.0518596e+10 -0.53984868 + 4700 0 1.0386105e+10 -0.53977803 + 4800 0 1.029525e+10 -0.53970882 + 4900 0 1.0519112e+10 -0.53969616 + 5000 0 1.0335841e+10 -0.53976477 +Loop time of 0.471663 on 4 procs for 5000 steps with 1024 atoms + +Performance: 0.092 tau/day, 10600.781 timesteps/s +95.8% CPU use with 4 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 0.067099 | 0.07105 | 0.077898 | 1.6 | 15.06 +Neigh | 0 | 0 | 0 | 0.0 | 0.00 +Comm | 0.0581 | 0.066084 | 0.072322 | 2.0 | 14.01 +Output | 0.0014644 | 0.002618 | 0.0037239 | 1.6 | 0.56 +Modify | 0.24817 | 0.25719 | 0.26697 | 1.3 | 54.53 +Other | | 0.07472 | | | 15.84 + +Nlocal: 256.000 ave 256 max 256 min +Histogram: 4 0 0 0 0 0 0 0 0 0 +Nghost: 105.000 ave 105 max 105 min +Histogram: 4 0 0 0 0 0 0 0 0 0 +Neighs: 544.000 ave 544 max 544 min +Histogram: 4 0 0 0 0 0 0 0 0 0 + +Total # of neighbors = 2176 +Ave neighs/atom = 2.1250000 +Neighbor list builds = 0 +Dangerous builds = 0 +reset_timestep 0 + + +# MSD +compute msd all msd + +thermo_style custom step temp epair c_msd[*] c_press + + +timestep 0.00001 +thermo 1000 + +# main run +run 20000 +Per MPI rank memory allocation (min/avg/max) = 5.427 | 5.427 | 5.427 Mbytes +Step Temp E_pair c_msd[1] c_msd[2] c_msd[3] c_msd[4] c_press + 0 1.0345945e+10 0 0 0 0 0 -0.53976477 + 1000 100114.28 0.0029703577 0.020320684 0.020950989 0 0.041271673 -0.43948247 + 2000 106825.83 0.020969054 0.039616412 0.039459167 0 0.079075578 -0.22765541 + 3000 105287.4 0.037343571 0.056828177 0.058639835 0 0.11546801 -0.11728136 + 4000 104522.23 0.052237136 0.080264931 0.080863543 0 0.16112847 0.033230576 + 5000 103277.94 0.053791862 0.099188864 0.10141444 0 0.20060331 0.073591503 + 6000 104252.87 0.073304776 0.11964238 0.1215576 0 0.24119999 0.22062305 + 7000 105184.19 0.089054043 0.13691291 0.14216478 0 0.27907769 0.29015692 + 8000 104211.82 0.072577918 0.15820522 0.15658491 0 0.31479013 0.25908291 + 9000 99242.172 0.071616004 0.17658708 0.17479704 0 0.35138412 0.26305532 + 10000 105070.83 0.077009979 0.20175025 0.19871513 0 0.40046538 0.34120567 + 11000 106421.07 0.098623061 0.22472634 0.22671582 0 0.45144216 0.44021335 + 12000 103209.85 0.12032847 0.25004966 0.25368441 0 0.50373406 0.57344873 + 13000 107156.89 0.1058386 0.27283231 0.2744873 0 0.54731961 0.47957408 + 14000 108119.3 0.1204768 0.29333677 0.30054535 0 0.59388213 0.51832639 + 15000 105477.62 0.12510026 0.32217621 0.32806599 0 0.6502422 0.50174158 + 16000 106676.27 0.10893618 0.34980866 0.36031184 0 0.7101205 0.44769198 + 17000 103048.41 0.10625673 0.3781797 0.37970499 0 0.75788468 0.42803898 + 18000 109454.8 0.10555778 0.40997694 0.41396777 0 0.82394471 0.41380982 + 19000 107459.73 0.11267582 0.43757738 0.43577856 0 0.87335594 0.4917748 + 20000 101991.9 0.084279008 0.45363612 0.46278076 0 0.91641688 0.41707912 +Loop time of 1.80877 on 4 procs for 20000 steps with 1024 atoms + +Performance: 9553.439 tau/day, 11057.221 timesteps/s +98.5% CPU use with 4 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 0.34461 | 0.36424 | 0.40948 | 4.4 | 20.14 +Neigh | 0.0031493 | 0.003215 | 0.0032432 | 0.1 | 0.18 +Comm | 0.19538 | 0.20419 | 0.2104 | 1.2 | 11.29 +Output | 0.00054121 | 0.00087297 | 0.0018425 | 0.0 | 0.05 +Modify | 0.98335 | 1.0156 | 1.0791 | 3.8 | 56.15 +Other | | 0.2207 | | | 12.20 + +Nlocal: 256.000 ave 261 max 252 min +Histogram: 1 1 0 0 0 0 1 0 0 1 +Nghost: 93.0000 ave 100 max 83 min +Histogram: 1 0 0 0 0 0 1 1 0 1 +Neighs: 662.250 ave 693 max 635 min +Histogram: 1 0 1 0 0 0 1 0 0 1 + +Total # of neighbors = 2649 +Ave neighs/atom = 2.5869141 +Neighbor list builds = 23 +Dangerous builds = 0 +Total wall time: 0:00:02 diff --git a/examples/USER/brownian/spherical_ABP/log.11May2021.in3d.ideal_apb.g++.1 b/examples/USER/brownian/spherical_ABP/log.11May2021.in3d.ideal_apb.g++.1 new file mode 100644 index 0000000000..0820471faa --- /dev/null +++ b/examples/USER/brownian/spherical_ABP/log.11May2021.in3d.ideal_apb.g++.1 @@ -0,0 +1,210 @@ +LAMMPS (8 Apr 2021) + using 1 OpenMP thread(s) per MPI task +# 3D overdamped active brownian dynamics with no interactions + +variable gamma_t string 3.0 +variable gamma_r string 1.0 +variable temp string 1.0 +variable seed equal 1974019 +variable fp string 4.0 +variable params string ${temp}_${gamma_t}_${gamma_r}_${fp} +variable params string 1.0_${gamma_t}_${gamma_r}_${fp} +variable params string 1.0_3.0_${gamma_r}_${fp} +variable params string 1.0_3.0_1.0_${fp} +variable params string 1.0_3.0_1.0_4.0 + +units lj +atom_style hybrid dipole sphere +WARNING: Atom style hybrid defines both, per-type and per-atom masses; both must be set, but only per-atom masses will be used (src/atom_vec_hybrid.cpp:156) +dimension 3 +newton off + +lattice sc 0.4 +Lattice spacing in x,y,z = 1.3572088 1.3572088 1.3572088 +region box block -8 8 -8 8 -8 8 +create_box 1 box +Created orthogonal box = (-10.857670 -10.857670 -10.857670) to (10.857670 10.857670 10.857670) + 1 by 1 by 1 MPI processor grid +create_atoms 1 box +Created 4096 atoms + create_atoms CPU = 0.004 seconds +mass * 1.0 +set type * dipole/random ${seed} 1.0 +set type * dipole/random 1974019 1.0 +Setting atom values ... + 4096 settings made for dipole/random +velocity all create 1.0 1 loop geom + +pair_style none + +# overdamped brownian dynamics time-step +fix step all brownian/sphere ${temp} ${seed} gamma_t ${gamma_t} gamma_r ${gamma_r} +fix step all brownian/sphere 1.0 ${seed} gamma_t ${gamma_t} gamma_r ${gamma_r} +fix step all brownian/sphere 1.0 1974019 gamma_t ${gamma_t} gamma_r ${gamma_r} +fix step all brownian/sphere 1.0 1974019 gamma_t 3.0 gamma_r ${gamma_r} +fix step all brownian/sphere 1.0 1974019 gamma_t 3.0 gamma_r 1.0 +# self-propulsion force along the dipole direction +fix activity all propel/self dipole ${fp} +fix activity all propel/self dipole 4.0 + +compute press all pressure NULL virial + +thermo_style custom step ke pe c_press + +#equilibration +timestep 0.0000000001 +thermo 100 +run 5000 +WARNING: No pairwise cutoff or binsize set. Atom sorting therefore disabled. (src/atom.cpp:2141) +WARNING: Communication cutoff is 0.0. No ghost atoms will be generated. Atoms may get lost. (src/comm_brick.cpp:167) +Per MPI rank memory allocation (min/avg/max) = 4.362 | 4.362 | 4.362 Mbytes +Step KinEng PotEng c_press + 0 1.4996338 0 0.068021726 + 100 5.184227e+09 0 0.06801544 + 200 5.2165482e+09 0 0.068010729 + 300 5.2782092e+09 0 0.068009058 + 400 5.3244927e+09 0 0.068003481 + 500 5.2376606e+09 0 0.067998237 + 600 5.2735634e+09 0 0.067998037 + 700 5.2692439e+09 0 0.068025402 + 800 5.2667984e+09 0 0.068030143 + 900 5.242057e+09 0 0.0680246 + 1000 5.2557468e+09 0 0.068028348 + 1100 5.2975687e+09 0 0.068029528 + 1200 5.2081927e+09 0 0.068017542 + 1300 5.2636873e+09 0 0.068012572 + 1400 5.2187907e+09 0 0.06802049 + 1500 5.2349541e+09 0 0.0680373 + 1600 5.216092e+09 0 0.068056885 + 1700 5.2598019e+09 0 0.068069504 + 1800 5.2569065e+09 0 0.068065306 + 1900 5.2072055e+09 0 0.068074863 + 2000 5.2092961e+09 0 0.068061619 + 2100 5.2918572e+09 0 0.068076418 + 2200 5.2680626e+09 0 0.068072149 + 2300 5.242958e+09 0 0.06806486 + 2400 5.2494099e+09 0 0.06805038 + 2500 5.2055798e+09 0 0.068072194 + 2600 5.2264829e+09 0 0.068069312 + 2700 5.3557342e+09 0 0.068064812 + 2800 5.2186177e+09 0 0.068042942 + 2900 5.2652497e+09 0 0.068044214 + 3000 5.1894899e+09 0 0.068044801 + 3100 5.241524e+09 0 0.068056675 + 3200 5.1915006e+09 0 0.06805641 + 3300 5.2367825e+09 0 0.068049946 + 3400 5.2288011e+09 0 0.068060182 + 3500 5.2704335e+09 0 0.068070881 + 3600 5.2886558e+09 0 0.068050439 + 3700 5.1976022e+09 0 0.068045927 + 3800 5.1525512e+09 0 0.068054494 + 3900 5.2212395e+09 0 0.068061432 + 4000 5.2309575e+09 0 0.068070842 + 4100 5.2260184e+09 0 0.068078378 + 4200 5.2829349e+09 0 0.068071652 + 4300 5.2204917e+09 0 0.068083072 + 4400 5.255242e+09 0 0.068066175 + 4500 5.2435681e+09 0 0.068050802 + 4600 5.2483356e+09 0 0.06805658 + 4700 5.2365098e+09 0 0.068041845 + 4800 5.2254325e+09 0 0.068038583 + 4900 5.1842852e+09 0 0.068028401 + 5000 5.2240722e+09 0 0.068031544 +Loop time of 5.14275 on 1 procs for 5000 steps with 4096 atoms + +Performance: 0.008 tau/day, 972.242 timesteps/s +99.9% CPU use with 1 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 0 | 0 | 0 | 0.0 | 0.00 +Neigh | 0 | 0 | 0 | 0.0 | 0.00 +Comm | 0.26842 | 0.26842 | 0.26842 | 0.0 | 5.22 +Output | 0.0035088 | 0.0035088 | 0.0035088 | 0.0 | 0.07 +Modify | 4.6588 | 4.6588 | 4.6588 | 0.0 | 90.59 +Other | | 0.212 | | | 4.12 + +Nlocal: 4096.00 ave 4096 max 4096 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Nghost: 817.000 ave 817 max 817 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Neighs: 0.00000 ave 0 max 0 min +Histogram: 1 0 0 0 0 0 0 0 0 0 + +Total # of neighbors = 0 +Ave neighs/atom = 0.0000000 +Neighbor list builds = 0 +Dangerous builds = 0 +reset_timestep 0 + +# MSD to demonstrate expected diffusive behaviour for ideal active +# brownian motion, which is +# +# MSD = (2*d*kb*T/gamma_t + 2*fp**2*gamma_r/(kb*T*gamma_t**2*(d-1)))*t +# + 2*fp**2*gamma_r**2/(gamma_t**2*(d-1)**2*(kb*T)**2)*(e^(-(d-1)*t*kb*T/gamma_r)-1) +# +# with d being simulation dimension +compute msd all msd + +thermo_style custom step ke pe c_msd[*] c_press + + +timestep 0.00001 +thermo 1000 + +# main run +run 12000 +WARNING: Communication cutoff is 0.0. No ghost atoms will be generated. Atoms may get lost. (src/comm_brick.cpp:167) +Per MPI rank memory allocation (min/avg/max) = 4.737 | 4.737 | 4.737 Mbytes +Step KinEng PotEng c_msd[1] c_msd[2] c_msd[3] c_msd[4] c_press + 0 5.2240722e+09 0 0 0 0 0 0.068031544 + 1000 52651.581 0 0.0066842466 0.0067977045 0.0066831353 0.020165086 0.060774985 + 2000 52835.806 0 0.013693443 0.014008773 0.013518945 0.041221161 0.094748037 + 3000 52097.629 0 0.020666918 0.021696789 0.020665685 0.063029392 0.10673866 + 4000 52579.452 0 0.028145318 0.028504548 0.02830967 0.084959536 0.13358122 + 5000 51255.456 0 0.035019271 0.034644123 0.03638843 0.10605182 0.13507609 + 6000 52730.035 0 0.041412307 0.042689213 0.043339117 0.12744064 0.16497663 + 7000 52247.642 0 0.048119396 0.050556395 0.050706527 0.14938232 0.16360301 + 8000 52169.849 0 0.055241196 0.058678631 0.059373122 0.17329295 0.1676169 + 9000 52520.526 0 0.063519587 0.066592779 0.066988842 0.19710121 0.17142694 + 10000 53519.297 0 0.07164814 0.074576535 0.075619236 0.22184391 0.15619444 + 11000 52937.293 0 0.077992504 0.083184462 0.082988794 0.24416576 0.15257327 + 12000 51762.283 0 0.085959749 0.090992292 0.08984213 0.26679417 0.15996211 +Loop time of 11.6748 on 1 procs for 12000 steps with 4096 atoms + +Performance: 888.063 tau/day, 1027.851 timesteps/s +99.9% CPU use with 1 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 0 | 0 | 0 | 0.0 | 0.00 +Neigh | 0.0024164 | 0.0024164 | 0.0024164 | 0.0 | 0.02 +Comm | 0.048127 | 0.048127 | 0.048127 | 0.0 | 0.41 +Output | 0.0019393 | 0.0019393 | 0.0019393 | 0.0 | 0.02 +Modify | 11.12 | 11.12 | 11.12 | 0.0 | 95.24 +Other | | 0.5027 | | | 4.31 + +Nlocal: 4096.00 ave 4096 max 4096 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Nghost: 0.00000 ave 0 max 0 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Neighs: 0.00000 ave 0 max 0 min +Histogram: 1 0 0 0 0 0 0 0 0 0 + +Total # of neighbors = 0 +Ave neighs/atom = 0.0000000 +Neighbor list builds = 75 +Dangerous builds = 0 + +# if you want to check that rotational diffusion is behaving as expected, +# uncomment next three lines for dump output and then plot , +# which should decay exponentially with timescale (d-1)*D_r (with d +# being simulation dimension) + +#dump 1 all custom 2000 dump_ideal_${params}_3d.lammpstrj id type # x y xu yu mux muy muz fx fy fz +#dump_modify 1 first yes sort id + +#run 120000 +Total wall time: 0:00:16 diff --git a/examples/USER/brownian/spherical_ABP/log.11May2021.in3d.ideal_apb.g++.4 b/examples/USER/brownian/spherical_ABP/log.11May2021.in3d.ideal_apb.g++.4 new file mode 100644 index 0000000000..c6e5b82090 --- /dev/null +++ b/examples/USER/brownian/spherical_ABP/log.11May2021.in3d.ideal_apb.g++.4 @@ -0,0 +1,210 @@ +LAMMPS (8 Apr 2021) + using 1 OpenMP thread(s) per MPI task +# 3D overdamped active brownian dynamics with no interactions + +variable gamma_t string 3.0 +variable gamma_r string 1.0 +variable temp string 1.0 +variable seed equal 1974019 +variable fp string 4.0 +variable params string ${temp}_${gamma_t}_${gamma_r}_${fp} +variable params string 1.0_${gamma_t}_${gamma_r}_${fp} +variable params string 1.0_3.0_${gamma_r}_${fp} +variable params string 1.0_3.0_1.0_${fp} +variable params string 1.0_3.0_1.0_4.0 + +units lj +atom_style hybrid dipole sphere +WARNING: Atom style hybrid defines both, per-type and per-atom masses; both must be set, but only per-atom masses will be used (src/atom_vec_hybrid.cpp:156) +dimension 3 +newton off + +lattice sc 0.4 +Lattice spacing in x,y,z = 1.3572088 1.3572088 1.3572088 +region box block -8 8 -8 8 -8 8 +create_box 1 box +Created orthogonal box = (-10.857670 -10.857670 -10.857670) to (10.857670 10.857670 10.857670) + 2 by 1 by 2 MPI processor grid +create_atoms 1 box +Created 4096 atoms + create_atoms CPU = 0.002 seconds +mass * 1.0 +set type * dipole/random ${seed} 1.0 +set type * dipole/random 1974019 1.0 +Setting atom values ... + 4096 settings made for dipole/random +velocity all create 1.0 1 loop geom + +pair_style none + +# overdamped brownian dynamics time-step +fix step all brownian/sphere ${temp} ${seed} gamma_t ${gamma_t} gamma_r ${gamma_r} +fix step all brownian/sphere 1.0 ${seed} gamma_t ${gamma_t} gamma_r ${gamma_r} +fix step all brownian/sphere 1.0 1974019 gamma_t ${gamma_t} gamma_r ${gamma_r} +fix step all brownian/sphere 1.0 1974019 gamma_t 3.0 gamma_r ${gamma_r} +fix step all brownian/sphere 1.0 1974019 gamma_t 3.0 gamma_r 1.0 +# self-propulsion force along the dipole direction +fix activity all propel/self dipole ${fp} +fix activity all propel/self dipole 4.0 + +compute press all pressure NULL virial + +thermo_style custom step ke pe c_press + +#equilibration +timestep 0.0000000001 +thermo 100 +run 5000 +WARNING: No pairwise cutoff or binsize set. Atom sorting therefore disabled. (src/atom.cpp:2141) +WARNING: Communication cutoff is 0.0. No ghost atoms will be generated. Atoms may get lost. (src/comm_brick.cpp:167) +Per MPI rank memory allocation (min/avg/max) = 4.319 | 4.319 | 4.319 Mbytes +Step KinEng PotEng c_press + 0 1.4996338 0 0.068021726 + 100 5.2484581e+09 0 0.068010768 + 200 5.277936e+09 0 0.068024039 + 300 5.2651427e+09 0 0.068014821 + 400 5.2066432e+09 0 0.068027583 + 500 5.2250276e+09 0 0.068030242 + 600 5.239771e+09 0 0.068018406 + 700 5.1953674e+09 0 0.068017081 + 800 5.2097107e+09 0 0.068010167 + 900 5.2559863e+09 0 0.068012923 + 1000 5.2743197e+09 0 0.068017855 + 1100 5.1999741e+09 0 0.068014189 + 1200 5.3216344e+09 0 0.068005604 + 1300 5.2839264e+09 0 0.067982558 + 1400 5.2462761e+09 0 0.067977843 + 1500 5.2208208e+09 0 0.067979594 + 1600 5.2740284e+09 0 0.067972573 + 1700 5.1919692e+09 0 0.067974452 + 1800 5.2497614e+09 0 0.067966417 + 1900 5.2910442e+09 0 0.067976096 + 2000 5.27238e+09 0 0.067963979 + 2100 5.3305398e+09 0 0.06795661 + 2200 5.205471e+09 0 0.067970212 + 2300 5.1803713e+09 0 0.067931775 + 2400 5.2134311e+09 0 0.067941825 + 2500 5.2367424e+09 0 0.067963456 + 2600 5.2246738e+09 0 0.067957556 + 2700 5.2514573e+09 0 0.067960724 + 2800 5.2601577e+09 0 0.067965167 + 2900 5.2422855e+09 0 0.067956561 + 3000 5.1796674e+09 0 0.067946764 + 3100 5.2308189e+09 0 0.067946585 + 3200 5.1835395e+09 0 0.067951909 + 3300 5.2762112e+09 0 0.067963199 + 3400 5.3224133e+09 0 0.067944918 + 3500 5.2314242e+09 0 0.06795318 + 3600 5.2760337e+09 0 0.067958005 + 3700 5.2549349e+09 0 0.06795228 + 3800 5.3343065e+09 0 0.067944561 + 3900 5.2440993e+09 0 0.067947433 + 4000 5.2565026e+09 0 0.067962624 + 4100 5.1766738e+09 0 0.067949542 + 4200 5.2058437e+09 0 0.067959946 + 4300 5.2777775e+09 0 0.067945883 + 4400 5.2020331e+09 0 0.067953495 + 4500 5.1417619e+09 0 0.067944161 + 4600 5.2672994e+09 0 0.067936777 + 4700 5.222847e+09 0 0.067943025 + 4800 5.2467842e+09 0 0.06794191 + 4900 5.2784378e+09 0 0.067939495 + 5000 5.2563969e+09 0 0.067940246 +Loop time of 1.55848 on 4 procs for 5000 steps with 4096 atoms + +Performance: 0.028 tau/day, 3208.260 timesteps/s +97.1% CPU use with 4 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 0 | 0 | 0 | 0.0 | 0.00 +Neigh | 0 | 0 | 0 | 0.0 | 0.00 +Comm | 0.15304 | 0.15677 | 0.16459 | 1.2 | 10.06 +Output | 0.0012078 | 0.0021182 | 0.0047011 | 3.2 | 0.14 +Modify | 1.1966 | 1.2236 | 1.2761 | 2.8 | 78.51 +Other | | 0.176 | | | 11.29 + +Nlocal: 1024.00 ave 1024 max 1024 min +Histogram: 4 0 0 0 0 0 0 0 0 0 +Nghost: 353.000 ave 353 max 353 min +Histogram: 4 0 0 0 0 0 0 0 0 0 +Neighs: 0.00000 ave 0 max 0 min +Histogram: 4 0 0 0 0 0 0 0 0 0 + +Total # of neighbors = 0 +Ave neighs/atom = 0.0000000 +Neighbor list builds = 0 +Dangerous builds = 0 +reset_timestep 0 + +# MSD to demonstrate expected diffusive behaviour for ideal active +# brownian motion, which is +# +# MSD = (2*d*kb*T/gamma_t + 2*fp**2*gamma_r/(kb*T*gamma_t**2*(d-1)))*t +# + 2*fp**2*gamma_r**2/(gamma_t**2*(d-1)**2*(kb*T)**2)*(e^(-(d-1)*t*kb*T/gamma_r)-1) +# +# with d being simulation dimension +compute msd all msd + +thermo_style custom step ke pe c_msd[*] c_press + + +timestep 0.00001 +thermo 1000 + +# main run +run 12000 +WARNING: Communication cutoff is 0.0. No ghost atoms will be generated. Atoms may get lost. (src/comm_brick.cpp:167) +Per MPI rank memory allocation (min/avg/max) = 4.694 | 4.694 | 4.694 Mbytes +Step KinEng PotEng c_msd[1] c_msd[2] c_msd[3] c_msd[4] c_press + 0 5.2563969e+09 0 0 0 0 0 0.067940246 + 1000 52568.549 0 0.0067249858 0.0066478843 0.0066014231 0.019974293 0.066777589 + 2000 52836.937 0 0.013611101 0.013799663 0.013161144 0.040571907 0.066769693 + 3000 52129.467 0 0.020360834 0.02089829 0.01995025 0.061209374 0.060026879 + 4000 52075.177 0 0.027638751 0.028062314 0.026895904 0.082596969 0.078290387 + 5000 52203.996 0 0.034087112 0.034933104 0.033832559 0.10285278 0.083657551 + 6000 52986.764 0 0.041562413 0.042238976 0.040542538 0.12434393 0.11542014 + 7000 51941.229 0 0.049216989 0.049250201 0.047598008 0.1460652 0.13739893 + 8000 52618.713 0 0.057198947 0.057409217 0.05404895 0.16865711 0.13681938 + 9000 52501.332 0 0.066447829 0.065262287 0.062271789 0.19398191 0.14306596 + 10000 52545.628 0 0.073800792 0.072510553 0.070100713 0.21641206 0.14689578 + 11000 52416.561 0 0.081881868 0.080638809 0.078969817 0.24149049 0.15608324 + 12000 52271.578 0 0.090521937 0.088555992 0.08592156 0.26499949 0.1474981 +Loop time of 3.13506 on 4 procs for 12000 steps with 4096 atoms + +Performance: 3307.113 tau/day, 3827.677 timesteps/s +99.2% CPU use with 4 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 0 | 0 | 0 | 0.0 | 0.00 +Neigh | 0.00060225 | 0.00060934 | 0.00061345 | 0.0 | 0.02 +Comm | 0.029197 | 0.029376 | 0.029582 | 0.1 | 0.94 +Output | 0.00060606 | 0.00087148 | 0.0016448 | 0.0 | 0.03 +Modify | 2.84 | 2.8773 | 2.8942 | 1.3 | 91.78 +Other | | 0.2269 | | | 7.24 + +Nlocal: 1024.00 ave 1037 max 999 min +Histogram: 1 0 0 0 0 0 0 1 1 1 +Nghost: 0.00000 ave 0 max 0 min +Histogram: 4 0 0 0 0 0 0 0 0 0 +Neighs: 0.00000 ave 0 max 0 min +Histogram: 4 0 0 0 0 0 0 0 0 0 + +Total # of neighbors = 0 +Ave neighs/atom = 0.0000000 +Neighbor list builds = 73 +Dangerous builds = 0 + +# if you want to check that rotational diffusion is behaving as expected, +# uncomment next three lines for dump output and then plot , +# which should decay exponentially with timescale (d-1)*D_r (with d +# being simulation dimension) + +#dump 1 all custom 2000 dump_ideal_${params}_3d.lammpstrj id type # x y xu yu mux muy muz fx fy fz +#dump_modify 1 first yes sort id + +#run 120000 +Total wall time: 0:00:04 diff --git a/examples/USER/cgsdk/sds-monolayer/in.sds-hybrid b/examples/USER/cgsdk/sds-monolayer/in.sds-hybrid index 3a9fcc13a9..eb3eca9c96 100644 --- a/examples/USER/cgsdk/sds-monolayer/in.sds-hybrid +++ b/examples/USER/cgsdk/sds-monolayer/in.sds-hybrid @@ -37,10 +37,11 @@ atom_modify first charged kspace_style pppm/cg 0.00001 kspace_modify order 3 -comm_modify mode multi neighbor 2.0 multi neigh_modify delay 4 every 2 check yes +comm_modify mode multi + timestep 10.0 fix 1 all nvt temp 310.0 310.0 100.0 diff --git a/examples/USER/fep/README.md b/examples/USER/fep/README.md index 416af526e7..d8f0935115 100644 --- a/examples/USER/fep/README.md +++ b/examples/USER/fep/README.md @@ -9,6 +9,9 @@ to the final states. * `CH4hyd` -- free energy of hydration of methane (simple). FEP and FDTI methods. +* `SPCEhyd` -- free energy of hydration of SPCE water (simple). FEP + in separate steps for the LJ sites and the atomic charges. + * `CH4-CF4` -- free energy difference of transforming methane into perfluoromethane, in water (quite simple). FEP and BAR methods. diff --git a/examples/USER/fep/SPCEhyd/README.md b/examples/USER/fep/SPCEhyd/README.md new file mode 100644 index 0000000000..de30f0c19b --- /dev/null +++ b/examples/USER/fep/SPCEhyd/README.md @@ -0,0 +1,45 @@ +Free Energy of Hydration of SPCE Water +====================================== + +Example calculation of the free energy of hydration of water with +LAMMPS using *compute fep*, *fix adapt/fep* and *pair lj/cut/soft*. + +The Lennard-Jones sites and the electrostatic charges are +created/annihilated in separate runs, which simplifies the use of +*fix adapt/fep* and *compute fep*. The Lennard-Jones sites are handled +using soft core potentials (*pair lj/cut/soft*). Trajectories are at +constant NpT, so corrections for the fluctuating volume are included. + +The following directories contain input files and results for +calculations using free-energy perturbation (FEP): + +* `mols` -- molecule description files and force field database used + to create the initial configuration used for the simulations + `data.lmp` + +* `fep01` -- Calculation using FEP, multi-stage creation of one SPC/E + molecule, LJ and q. Results in `fep01-lj.fep` and `fep01-lj.fep` + +* `fep10` -- Calculation using FEP, multi-stage deletion of one SPC/E + molecule, q and LJ. Results in `fep10-q.fep` and `fep10-lj.fep` + +The Python script `fep.py` found in the +`tools` directory can be used to calculate the free-energy differences +corresponding to the transformations above: + + fep.py 300 < fep01-lj.fep + + fep.py 300 < fep01-q.fep + + fep.py 300 < fep10-q.fep + + fep.py 300 < fep10-lj.fep + +The outputs are in kcal/mol and can be compared with the experimental +value of -6.3 kcal/mol, or with a simulation value from the literature +of -6.7 kcal/mol +[Gonçalves, Stassen, Pure Appl Chem 76 (2004) 231](https://doi.org/10.1351/pac200476010231). + +These example calculations are for tutorial purposes only. The results +may not be of research quality (not enough sampling, size of the step +in lambda not optimized, etc.) diff --git a/examples/USER/fep/SPCEhyd/fep01/fep01-lj.fep b/examples/USER/fep/SPCEhyd/fep01/fep01-lj.fep new file mode 100644 index 0000000000..67dc4bfd8d --- /dev/null +++ b/examples/USER/fep/SPCEhyd/fep01/fep01-lj.fep @@ -0,0 +1,22 @@ +# Time-averaged data for fix FEP +# TimeStep c_FEP[1] c_FEP[2] c_FEP[3] +100000 0.00203701 17656.7 17717 +200000 0.0148799 17235.9 17669.1 +300000 0.0478517 16378 17736.4 +400000 0.100195 15034.8 17725.7 +500000 0.218866 12390.7 17698.7 +600000 0.371395 9907.37 17731.1 +700000 0.613593 7317.6 17750.5 +800000 0.895639 5984.39 17714.3 +900000 0.638838 8921.42 17726.2 +1000000 0.372214 11782.7 17728.6 +1100000 0.295555 12613 17730.3 +1200000 0.152975 15013.7 17792.5 +1300000 0.106403 15927.4 17737.2 +1400000 0.0570002 16875.1 17725.9 +1500000 0.0182185 17924.9 17790.8 +1600000 -0.0319612 19263.7 17775.1 +1700000 -0.0684472 20338.7 17786.9 +1800000 -0.0941487 21136.1 17764.8 +1900000 -0.132444 22374 17750.6 +2000000 -0.153928 23156.8 17744.8 diff --git a/examples/USER/fep/SPCEhyd/fep01/fep01-lj.out b/examples/USER/fep/SPCEhyd/fep01/fep01-lj.out new file mode 100644 index 0000000000..3eaf2fd98c --- /dev/null +++ b/examples/USER/fep/SPCEhyd/fep01/fep01-lj.out @@ -0,0 +1,592 @@ +LAMMPS (29 Oct 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (src/lammps/src/comm.cpp:94) + using 1 OpenMP thread(s) per MPI task +Reading data file ... + orthogonal box = (0.0000000 0.0000000 0.0000000) to (29.204526 29.204526 29.204526) + 2 by 2 by 3 MPI processor grid + reading atoms ... + 1800 atoms + scanning bonds ... + 1 = max bonds/atom + scanning angles ... + 1 = max angles/atom + reading bonds ... + 1200 bonds + reading angles ... + 600 angles +Finding 1-2 1-3 1-4 neighbors ... + special bond factors lj: 0 0 0.5 + special bond factors coul: 0 0 0.5 + 2 = max # of 1-2 neighbors + 1 = max # of 1-3 neighbors + 1 = max # of 1-4 neighbors + 2 = max # of special neighbors + special bonds CPU = 0.001 seconds + read_data CPU = 0.018 seconds +Finding SHAKE clusters ... + 0 = # of size 2 clusters + 600 = # of size 3 clusters + 0 = # of size 4 clusters + 0 = # of frozen angles + find clusters CPU = 0.001 seconds +Setting atom values ... + 3 settings made for charge +PPPM initialization ... + using 12-bit tables for long-range coulomb (src/lammps/src/kspace.cpp:339) + G vector (1/distance) = 0.25066829 + grid = 20 20 20 + stencil order = 5 + estimated absolute RMS force accuracy = 0.0030719151 + estimated relative force accuracy = 9.250981e-06 + using double precision FFTW3 + 3d grid and FFT values/proc = 3757 800 +Neighbor list info ... + update every 1 steps, delay 10 steps, check yes + max neighbors/atom: 2000, page size: 100000 + master list distance cutoff = 14 + ghost atom cutoff = 14 + binsize = 7, bins = 5 5 5 + 3 neighbor lists, perpetual/occasional/extra = 3 0 0 + (1) pair lj/cut/coul/long, perpetual, skip from (3) + attributes: half, newton on + pair build: skip + stencil: none + bin: none + (2) pair lj/cut/soft, perpetual, skip from (3) + attributes: half, newton on + pair build: skip + stencil: none + bin: none + (3) neighbor class addition, perpetual + attributes: half, newton on + pair build: half/bin/newton + stencil: half/bin/3d/newton + bin: standard +Setting up Verlet run ... + Unit style : real + Current step : 0 + Time step : 1 +Per MPI rank memory allocation (min/avg/max) = 9.477 | 9.489 | 9.507 Mbytes +Step CPU TotEng KinEng PotEng E_vdwl E_coul E_long Temp Press Volume Density + 0 0 1344.7207 1251.046 93.67475 54.469999 30049.554 -30010.349 300 10284.691 24908.667 0.72058313 + 5000 28.694564 -5486.1043 1288.3881 -6774.4924 1379.4864 21688.643 -30296.277 308.95461 54.310852 17485.125 1.0265163 + 10000 60.154999 -5445.984 1289.4468 -6735.4308 1414.0583 21715.025 -30296.898 309.2085 479.66554 17890.87 1.003236 + 15000 93.146023 -5513.8313 1247.6759 -6761.5072 1299.4795 21790.371 -30301.687 299.19185 -508.0062 17325.35 1.0359828 + 20000 125.27418 -5449.2886 1284.4959 -6733.7844 1413.7763 21713.785 -30296.156 308.02126 590.93324 17813.863 1.0075729 + 25000 159.91521 -5488.9702 1263.558 -6752.5282 1419.1433 21691.575 -30298.164 303.00038 745.03072 17529.113 1.0239403 + 30000 192.99516 -5560.7692 1261.1821 -6821.9512 1417.204 21619.369 -30296.506 302.43062 247.03569 17579.175 1.0210243 + 35000 226.31455 -5475.9339 1217.5826 -6693.5164 1416.6051 21740.37 -30296.23 291.97549 585.78922 17897.211 1.0028805 + 40000 257.38113 -5547.1985 1242.3037 -6789.5022 1368.836 21696.099 -30299.114 297.90361 -308.25384 17644.122 1.017266 + 45000 290.1833 -5546.482 1247.5586 -6794.0405 1478.3456 21579.538 -30298.856 299.16372 1088.2882 17592.834 1.0202316 + 50000 323.27342 -5587.1305 1245.2866 -6832.4171 1494.5555 21518.708 -30301.175 298.6189 1294.6268 17477.389 1.0269707 + 55000 357.01676 -5579.512 1240.4584 -6819.9704 1395.6861 21642.898 -30295.857 297.46109 -99.93318 17770.807 1.0100141 + 60000 388.82447 -5489.9387 1234.3341 -6724.2728 1285.7293 21870.543 -30295.028 295.99251 -1147.8529 17885.706 1.0035257 + 65000 422.87575 -5552.7156 1212.0793 -6764.7949 1333.565 21754.173 -30298.11 290.65582 -712.18036 17669.638 1.015797 + 70000 457.6093 -5497.6045 1258.538 -6756.1425 1322.4574 21754.169 -30297.784 301.79659 -688.38797 17616.532 1.0188592 + 75000 491.97196 -5625.4891 1179.169 -6804.6581 1390.1744 21637.212 -30297.753 282.76395 42.978393 17500.681 1.0256038 + 80000 525.96216 -5522.0997 1232.6571 -6754.7568 1341.1534 21770.924 -30301.663 295.59035 -265.75148 17495.977 1.0258796 + 85000 559.772 -5429.8519 1249.9284 -6679.7803 1347.2093 21813.241 -30297.861 299.732 -111.81756 17957.604 0.9995078 + 90000 593.75975 -5560.9592 1265.0979 -6826.0571 1463.6326 21567.748 -30297.643 303.36965 599.30412 17830.144 1.0066528 + 95000 627.953 -5544.2442 1277.7084 -6821.9526 1407.1904 21633.729 -30299.133 306.39364 192.51289 17668.067 1.0158873 + 100000 662.08715 -5474.8467 1286.3434 -6761.1902 1377.3305 21690.898 -30299.316 308.46431 -161.00615 17888.012 1.0033963 +Loop time of 662.087 on 12 procs for 100000 steps with 1800 atoms + +Performance: 13.050 ns/day, 1.839 hours/ns, 151.038 timesteps/s +94.9% CPU use with 12 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 345.52 | 374.93 | 410.41 | 88.0 | 56.63 +Bond | 1.2655 | 1.3414 | 1.443 | 5.2 | 0.20 +Kspace | 99.52 | 135.77 | 165.42 | 148.0 | 20.51 +Neigh | 28.904 | 28.95 | 28.997 | 0.6 | 4.37 +Comm | 41.498 | 45.503 | 50.663 | 51.8 | 6.87 +Output | 0.0013665 | 0.0014183 | 0.0018614 | 0.4 | 0.00 +Modify | 49.685 | 64.033 | 72.601 | 117.1 | 9.67 +Other | | 11.56 | | | 1.75 + +Nlocal: 150.000 ave 169 max 133 min +Histogram: 1 2 1 1 2 1 0 3 0 1 +Nghost: 6036.17 ave 6092 max 5976 min +Histogram: 3 1 0 0 0 1 3 2 0 2 +Neighs: 86433.7 ave 96904 max 78425 min +Histogram: 3 1 1 0 2 1 1 0 1 2 + +Total # of neighbors = 1037204 +Ave neighs/atom = 576.22444 +Ave special neighs/atom = 2.0000000 +Neighbor list builds = 4498 +Dangerous builds = 9 +PPPM initialization ... + using 12-bit tables for long-range coulomb (src/lammps/src/kspace.cpp:339) + G vector (1/distance) = 0.25099061 + grid = 18 18 18 + stencil order = 5 + estimated absolute RMS force accuracy = 0.0035434233 + estimated relative force accuracy = 1.0670914e-05 + using double precision FFTW3 + 3d grid and FFT values/proc = 3328 648 +FEP settings ... + temperature = 300.000000 + tail no + lj/cut/soft lambda 1-2 3-4 +Neighbor list info ... + update every 1 steps, delay 10 steps, check yes + max neighbors/atom: 2000, page size: 100000 + master list distance cutoff = 14 + ghost atom cutoff = 14 + binsize = 7, bins = 4 4 4 + 3 neighbor lists, perpetual/occasional/extra = 3 0 0 + (1) pair lj/cut/coul/long, perpetual, skip from (3) + attributes: half, newton on + pair build: skip + stencil: none + bin: none + (2) pair lj/cut/soft, perpetual, skip from (3) + attributes: half, newton on + pair build: skip + stencil: none + bin: none + (3) neighbor class addition, perpetual + attributes: half, newton on + pair build: half/bin/newton + stencil: half/bin/3d/newton + bin: standard +Setting up Verlet run ... + Unit style : real + Current step : 0 + Time step : 1 +Per MPI rank memory allocation (min/avg/max) = 9.473 | 9.524 | 9.885 Mbytes +Step TotEng KinEng PotEng E_vdwl E_coul E_long Temp Press Density v_lambda + 0 -5474.8313 1286.3434 -6761.1748 1377.3305 21729.814 -30338.216 308.46431 -145.89091 1.0033963 0 + 5000 -5501.3742 1227.5839 -6728.9581 1383.6091 21832.227 -30338.792 294.37381 197.29783 1.0066092 0.0025 + 10000 -5464.6748 1299.6003 -6764.2751 1422.869 21700.714 -30339.338 311.64329 911.1393 1.0308533 0.005 + 15000 -5570.8381 1254.6639 -6825.502 1432.6295 21661.321 -30337.889 300.86757 606.08029 1.0232505 0.0075 + 20000 -5517.9977 1279.1 -6797.0976 1408.873 21713.571 -30334.986 306.72733 71.014518 1.00536 0.01 + 25000 -5502.8662 1256.1509 -6759.0171 1354.4185 21819.904 -30335.966 301.22416 -280.3804 1.0121715 0.0125 + 30000 -5479.0987 1247.8675 -6726.9662 1413.5362 21734.153 -30338.534 299.23779 347.7292 1.0037264 0.015 + 35000 -5507.6469 1301.4131 -6809.06 1460.033 21614.564 -30336.109 312.07801 771.56011 1.0011707 0.0175 + 40000 -5487.2955 1295.7159 -6783.0113 1321.8494 21814.192 -30340.297 310.71181 -399.68468 1.0295009 0.02 + 45000 -5582.7908 1203.9957 -6786.7865 1428.232 21690.272 -30335.2 288.71737 516.0938 1.0238261 0.0225 + 50000 -5543.0112 1239.5077 -6782.5189 1442.8695 21660.077 -30336.998 297.23314 966.89878 1.0234172 0.025 + 55000 -5590.3604 1232.4497 -6822.8101 1322.5893 21772.961 -30338.359 295.54062 -790.54432 1.0210518 0.0275 + 60000 -5479.6266 1218.6941 -6698.3207 1363.233 21839.328 -30336.264 292.24204 102.67075 1.0069364 0.03 + 65000 -5492.8433 1277.3976 -6770.2409 1361.2794 21754.915 -30335.072 306.31909 -345.94035 1.0111736 0.0325 + 70000 -5564.9632 1256.2689 -6821.2322 1418.089 21663.812 -30334.208 301.25246 -235.62062 1.0000081 0.035 + 75000 -5431.4667 1233.6963 -6665.163 1385.2972 21867.508 -30337.413 295.83957 605.68584 1.015264 0.0375 + 80000 -5560.6719 1236.9415 -6797.6134 1374.1241 21748.636 -30335.156 296.61776 -202.53528 1.015924 0.04 + 85000 -5636.8905 1233.835 -6870.7255 1480.1666 21576.608 -30340.174 295.87281 827.4631 1.0212177 0.0425 + 90000 -5551.2917 1222.5755 -6773.8672 1452.5674 21638.206 -30335.595 293.1728 661.99097 1.0090475 0.045 + 95000 -5543.8224 1249.9355 -6793.758 1396.082 21724.933 -30339.764 299.73372 201.87719 1.0163116 0.0475 + 100000 -5558.5664 1227.1814 -6785.7479 1463.1907 21666.038 -30337.879 294.2773 955.6758 1.0158894 0.05 + 105000 -5507.0698 1233.0098 -6740.0796 1356.1635 21774.574 -30334.594 295.67493 -359.01022 1.0056685 0.0525 + 110000 -5561.0676 1221.0774 -6782.145 1403.5966 21692.913 -30337.463 292.81355 -87.113696 1.0133037 0.055 + 115000 -5526.6141 1243.9541 -6770.5682 1475.7194 21626.703 -30337.179 298.29937 855.85038 1.0118327 0.0575 + 120000 -5555.6741 1253.3796 -6809.0538 1346.5604 21755.315 -30340.743 300.55961 -1079.5856 0.98312027 0.06 + 125000 -5620.4084 1221.0146 -6841.423 1406.8142 21677.116 -30335.053 292.7985 -61.600379 1.0173627 0.0625 + 130000 -5586.4001 1204.2423 -6790.6425 1352.9098 21770.892 -30339.505 288.77651 -460.92375 1.0164837 0.065 + 135000 -5568.5186 1239.2748 -6807.7934 1401.8615 21748.687 -30340.555 297.17728 371.66677 1.0241473 0.0675 + 140000 -5526.4808 1249.0145 -6775.4953 1424.2185 21664.576 -30340.16 299.51285 520.63161 1.0238118 0.07 + 145000 -5532.8733 1254.9515 -6787.8248 1379.0629 21737.241 -30337.358 300.93654 -64.517937 1.0105756 0.0725 + 150000 -5549.4332 1228.3832 -6777.8163 1428.5036 21702.196 -30337.055 294.56547 363.42307 1.0052127 0.075 + 155000 -5547.9284 1256.6973 -6804.6257 1379.9059 21727.03 -30338.382 301.35519 35.503942 1.0222903 0.0775 + 160000 -5481.1383 1320.4344 -6801.5727 1390.3369 21729.299 -30337.062 316.63929 50.667563 1.0171266 0.08 + 165000 -5560.7321 1247.5004 -6808.2325 1422.2771 21662.917 -30341.024 299.14978 56.116137 1.0020068 0.0825 + 170000 -5541.3469 1257.9624 -6799.3094 1381.0133 21739.612 -30336.303 301.65856 -176.83515 1.0084433 0.085 + 175000 -5481.784 1239.9656 -6721.7496 1366.9952 21834.266 -30335.454 297.34293 377.28019 1.0297034 0.0875 + 180000 -5609.1447 1180.414 -6789.5587 1410.9273 21683.391 -30338.821 283.06249 203.46781 1.013427 0.09 + 185000 -5508.6637 1237.4229 -6746.0866 1401.5929 21736.318 -30334.806 296.7332 287.44715 1.0114427 0.0925 + 190000 -5524.666 1249.3421 -6774.008 1414.6887 21711.574 -30338.223 299.5914 108.63857 0.99681381 0.095 + 195000 -5613.4475 1179.1603 -6792.6078 1415.5702 21704.669 -30335.058 282.76187 -4.252137 1.0025742 0.0975 + 200000 -5577.1744 1219.3392 -6796.5136 1358.774 21688.932 -30339.219 292.39674 -339.31655 1.0272643 0.1 + 205000 -5473.5241 1291.8332 -6765.3573 1364.0148 21747.958 -30335.578 309.78075 -140.98482 1.0134706 0.1025 + 210000 -5412.6987 1304.0334 -6716.7321 1399.175 21790.687 -30338.388 312.70635 475.62089 1.0077929 0.105 + 215000 -5498.5445 1272.4319 -6770.9765 1305.2038 21853.899 -30334.099 305.12834 -964.41064 1.0117303 0.1075 + 220000 -5552.46 1236.8433 -6789.3033 1413.5383 21633.766 -30337.007 296.59421 126.59413 1.0135386 0.11 + 225000 -5463.5199 1281.9539 -6745.4738 1391.2426 21760.978 -30337.489 307.41171 372.98508 1.0154691 0.1125 + 230000 -5556.2415 1213.9742 -6770.2157 1373.9439 21746.355 -30331.29 291.11021 310.03784 1.0411894 0.115 + 235000 -5468.6139 1256.2727 -6724.8866 1419.9068 21755.903 -30337.057 301.25335 446.80106 1.008653 0.1175 + 240000 -5505.2272 1297.8543 -6803.0815 1389.9299 21721.555 -30336.374 311.22461 -216.33356 1.0007317 0.12 + 245000 -5556.4175 1231.9791 -6788.3966 1385.1734 21697.589 -30334.804 295.42778 -115.65689 1.0127289 0.1225 + 250000 -5570.5583 1261.1376 -6831.6959 1404.3069 21667.366 -30337.205 302.41997 -90.154201 1.0161797 0.125 + 255000 -5581.0101 1222.6213 -6803.6314 1393.6905 21687.538 -30336.542 293.18377 -58.781363 1.013542 0.1275 + 260000 -5552.6506 1225.5794 -6778.2299 1446.9969 21648.822 -30336.275 293.89312 749.70391 1.0136121 0.13 + 265000 -5527.9133 1245.5123 -6773.4255 1328.166 21792.912 -30336.854 298.67302 -825.99884 1.0063806 0.1325 + 270000 -5565.2798 1266.7326 -6832.0124 1488.6605 21582.46 -30339.347 303.76165 1592.7435 1.0374567 0.135 + 275000 -5567.4599 1211.9673 -6779.4272 1306.1895 21846.2 -30334.296 290.62895 -1062.6911 1.0107451 0.1375 + 280000 -5456.1965 1291.476 -6747.6725 1324.4174 21839.645 -30339.854 309.69509 -531.26163 1.0140474 0.14 + 285000 -5561.638 1229.3427 -6790.9808 1353.7161 21794.932 -30334.258 294.79557 -269.97414 1.0110394 0.1425 + 290000 -5486.1623 1276.1413 -6762.3036 1380.4775 21777.115 -30336.678 306.01784 -56.691362 1.0105051 0.145 + 295000 -5531.2556 1253.1859 -6784.4415 1354.9857 21750.76 -30338.549 300.51314 -555.4993 1.0159961 0.1475 + 300000 -5583.1864 1250.0209 -6833.2073 1431.4702 21654.547 -30337.557 299.75419 360.48799 1.0226237 0.15 + 305000 -5492.9399 1263.2527 -6756.1925 1385.9115 21749.132 -30337.333 302.92716 -94.633279 1.0008411 0.1525 + 310000 -5583.9951 1217.2631 -6801.2582 1439.2175 21660.614 -30332.833 291.89889 575.48041 1.0264848 0.155 + 315000 -5554.3025 1247.1517 -6801.4542 1305.5379 21807.382 -30337.478 299.06616 -1206.257 1.0085908 0.1575 + 320000 -5596.1914 1233.0393 -6829.2307 1431.9073 21667.612 -30337.859 295.682 745.35002 1.0291 0.16 + 325000 -5532.0063 1245.6006 -6777.6069 1479.6102 21640.94 -30338.346 298.69419 1390.0859 1.0311996 0.1625 + 330000 -5511.0183 1236.8356 -6747.854 1332.4766 21826.148 -30335.853 296.59237 -346.80349 1.020578 0.165 + 335000 -5512.3188 1238.1769 -6750.4957 1324.5201 21836.682 -30338.607 296.91401 -457.67242 1.0274241 0.1675 + 340000 -5504.0924 1242.6354 -6746.7278 1432.7958 21708.299 -30333.659 297.98314 674.26615 1.0055018 0.17 + 345000 -5522.5796 1265.0266 -6787.6062 1382.0078 21743.218 -30338.398 303.35254 -151.81029 1.0067371 0.1725 + 350000 -5545.7535 1267.9089 -6813.6624 1404.8847 21674.34 -30337.099 304.04372 128.71982 1.0135501 0.175 + 355000 -5499.0073 1275.2358 -6774.2431 1484.721 21658.719 -30337.688 305.80071 1468.7411 1.0346991 0.1775 + 360000 -5540.4144 1242.034 -6782.4484 1440.0683 21684.985 -30338.578 297.83892 662.99982 1.0085045 0.18 + 365000 -5472.9692 1267.5044 -6740.4736 1350.7945 21806.742 -30338.576 303.94672 -303.53857 1.0149794 0.1825 + 370000 -5563.7998 1257.4972 -6821.297 1363.9578 21745.681 -30339.039 301.54699 -616.39894 1.0015708 0.185 + 375000 -5494.3434 1204.4931 -6698.8365 1308.4989 21883.359 -30336.609 288.83665 -1009.8181 0.99179087 0.1875 + 380000 -5516.3392 1223.7518 -6740.0911 1355.9449 21816.149 -30337.588 293.45488 36.407556 1.02042 0.19 + 385000 -5534.8879 1226.8136 -6761.7015 1433.131 21698.233 -30336.201 294.18909 1224.3283 1.0416014 0.1925 + 390000 -5506.0676 1262.1264 -6768.194 1449.4667 21692.815 -30334.182 302.65707 932.23514 1.0216439 0.195 + 395000 -5601.9693 1234.2373 -6836.2066 1425.5787 21658.64 -30337.816 295.96929 254.00155 1.0148905 0.1975 + 400000 -5523.9324 1254.811 -6778.7434 1371.5302 21763.333 -30334.914 300.90286 -41.512594 1.0129399 0.2 + 405000 -5615.7822 1209.5514 -6825.3336 1431.7105 21604.804 -30336.544 290.04963 385.86176 1.02069 0.2025 + 410000 -5564.9317 1238.9308 -6803.8625 1334.6011 21725.649 -30342.03 297.09479 -668.14712 1.0114204 0.205 + 415000 -5565.723 1201.4165 -6767.1395 1407.6153 21731.714 -30339.794 288.09888 284.39352 1.0181526 0.2075 + 420000 -5514.1574 1259.6967 -6773.8542 1397.8555 21765.77 -30335.098 302.07445 -93.583513 1.003494 0.21 + 425000 -5566.4826 1233.6609 -6800.1435 1354.6757 21745.391 -30337.658 295.83106 -543.91774 1.0113019 0.2125 + 430000 -5544.9786 1227.5367 -6772.5153 1391.971 21756.332 -30337.075 294.36248 -52.853646 1.0029339 0.215 + 435000 -5457.9703 1243.8289 -6701.7992 1355.8144 21842.63 -30336.263 298.26936 96.07031 1.0162599 0.2175 + 440000 -5444.1352 1261.5679 -6705.7031 1327.5589 21849.174 -30337.667 302.52315 -159.33457 1.0201548 0.22 + 445000 -5569.3089 1253.1592 -6822.4681 1450.3256 21615.072 -30333.867 300.50675 497.07257 1.0162108 0.2225 + 450000 -5499.4632 1276.3744 -6775.8376 1436.3419 21681.852 -30337.613 306.07374 416.13585 1.0087874 0.225 + 455000 -5527.7917 1199.0123 -6726.804 1371.2866 21773.376 -30335.146 287.52235 -330.60298 0.99153512 0.2275 + 460000 -5634.9316 1214.1147 -6849.0463 1471.6124 21618.634 -30339.723 291.14391 831.79551 1.0154836 0.23 + 465000 -5492.5994 1247.2074 -6739.8067 1425.1188 21720.998 -30333.086 299.0795 782.63379 1.0210767 0.2325 + 470000 -5528.6598 1242.3825 -6771.0422 1413.2305 21674.806 -30339.41 297.9225 371.75935 1.018354 0.235 + 475000 -5570.6679 1274.1035 -6844.7714 1423.8596 21624.648 -30335.904 305.52919 243.67178 1.0157084 0.2375 + 480000 -5487.1119 1241.0548 -6728.1667 1421.7563 21732.588 -30338.933 297.60413 814.07568 1.0255732 0.24 + 485000 -5575.4172 1242.9613 -6818.3785 1402.7921 21677.417 -30338.336 298.0613 20.597793 1.0229831 0.2425 + 490000 -5578.303 1221.4158 -6799.7188 1430.5395 21694.761 -30336.572 292.89469 430.02647 1.0131845 0.245 + 495000 -5573.1773 1222.1238 -6795.3011 1502.7203 21609.264 -30338.892 293.06448 1494.9905 1.0226527 0.2475 + 500000 -5470.7153 1208.6593 -6679.3745 1401.9799 21819.612 -30335.747 289.83569 552.82762 1.003873 0.25 + 505000 -5626.3096 1246.8303 -6873.1399 1504.3826 21517.763 -30336.928 298.98909 1008.6542 1.0255205 0.2525 + 510000 -5562.1313 1242.0291 -6804.1604 1360.6445 21745.689 -30334.283 297.83775 -300.55499 1.021938 0.255 + 515000 -5558.6681 1241.9812 -6800.6493 1399.186 21683.627 -30339.266 297.82628 -293.22181 1.0035268 0.2575 + 520000 -5450.6773 1276.8169 -6727.4943 1246.5956 21918.914 -30335.226 306.17986 -1527.0713 1.0017384 0.26 + 525000 -5495.0769 1260.1524 -6755.2294 1413.8065 21708.663 -30338.153 302.18372 193.49778 1.0090792 0.2625 + 530000 -5519.0236 1258.9944 -6778.018 1423.1288 21726.251 -30338.72 301.90602 402.81349 1.0058563 0.265 + 535000 -5523.3166 1265.8203 -6789.1369 1403.2616 21669.617 -30333.964 303.54286 -212.90176 1.0022662 0.2675 + 540000 -5469.6354 1267.2634 -6736.8988 1354.6136 21815.059 -30334.909 303.88893 -143.78705 1.0102562 0.27 + 545000 -5475.0926 1332.0416 -6807.1342 1362.668 21728.374 -30335.495 319.42269 -408.94813 1.008036 0.2725 + 550000 -5604.8235 1220.5785 -6825.4021 1466.9259 21609.628 -30340.185 292.69392 562.0671 0.99832249 0.275 + 555000 -5546.9498 1194.3285 -6741.2783 1324.6404 21805.776 -30334.506 286.39919 -831.67066 1.0068815 0.2775 + 560000 -5505.8852 1231.4127 -6737.2978 1403.1199 21743.566 -30336.518 295.29195 134.18023 1.0068652 0.28 + 565000 -5620.5688 1225.712 -6846.2808 1418.5638 21624.884 -30336.305 293.92493 -45.194557 1.0148173 0.2825 + 570000 -5522.2261 1265.1646 -6787.3907 1431.1348 21666.003 -30337.479 303.38564 470.11582 1.0164271 0.285 + 575000 -5586.7521 1277.3002 -6864.0523 1411.585 21610.013 -30338.867 306.29575 3.3115648 1.0213675 0.2875 + 580000 -5612.9476 1236.9792 -6849.9268 1385.7307 21698.533 -30334.246 296.62679 -180.75952 1.0145947 0.29 + 585000 -5564.6816 1225.6381 -6790.3197 1333.2129 21796.701 -30335.104 293.90721 -690.26993 1.0116307 0.2925 + 590000 -5505.6734 1225.965 -6731.6383 1388.9635 21758.959 -30333.889 293.98559 227.32836 1.0145465 0.295 + 595000 -5547.8043 1248.8277 -6796.632 1360.0327 21712.026 -30336.566 299.46806 -375.66988 1.0129675 0.2975 + 600000 -5531.1317 1226.5624 -6757.6941 1288.9733 21854.05 -30334.778 294.12886 -869.81336 1.0198957 0.3 + 605000 -5491.3585 1261.835 -6753.1935 1362.4507 21774.232 -30332.849 302.58721 -165.60547 1.0153308 0.3025 + 610000 -5614.7624 1197.5896 -6812.352 1337.5281 21753.381 -30338.757 287.1812 -750.69767 1.0108767 0.305 + 615000 -5576.4304 1197.4926 -6773.9229 1360.9065 21756.815 -30335.31 287.15793 -693.56508 0.99901143 0.3075 + 620000 -5548.9351 1231.926 -6780.861 1367.5138 21765.942 -30335.526 295.41503 -447.87437 1.0071454 0.31 + 625000 -5478.0177 1304.9593 -6782.9771 1472.1656 21630.028 -30337.864 312.92838 902.38662 1.0047677 0.3125 + 630000 -5488.216 1246.0606 -6734.2767 1409.7407 21771.095 -30336.403 298.80451 480.81355 1.0089233 0.315 + 635000 -5539.378 1232.3758 -6771.7538 1370.1156 21731.482 -30335.377 295.52291 -713.31325 0.98843736 0.3175 + 640000 -5432.8862 1296.4151 -6729.3012 1343.8929 21825.715 -30336.397 310.87948 -397.39803 1.0014659 0.32 + 645000 -5588.6649 1238.8032 -6827.4681 1428.9906 21666.562 -30335.249 297.06419 599.51702 1.0293674 0.3225 + 650000 -5519.3466 1230.0381 -6749.3847 1453.0003 21705.846 -30339.941 294.96233 1124.8857 1.026394 0.325 + 655000 -5533.8266 1258.3233 -6792.1499 1371.5049 21733.113 -30338.134 301.74509 -435.07103 0.99919968 0.3275 + 660000 -5528.4015 1269.4116 -6797.8131 1338.761 21729.189 -30334.098 304.40406 -562.90491 1.0192565 0.33 + 665000 -5545.4581 1249.7274 -6795.1854 1412.241 21718.149 -30338.751 299.6838 275.8861 1.0143558 0.3325 + 670000 -5565.5948 1226.9318 -6792.5266 1315.7564 21812.516 -30339.437 294.21745 -830.3758 1.0195327 0.335 + 675000 -5517.0516 1226.7374 -6743.789 1415.6314 21750.702 -30335.553 294.17081 527.33474 1.0176545 0.3375 + 680000 -5545.2799 1223.2674 -6768.5473 1384.1095 21790.646 -30337.184 293.33872 -129.6713 1.0060356 0.34 + 685000 -5498.7222 1275.9994 -6774.7216 1415.2483 21718.812 -30334.721 305.98383 159.13649 1.0044781 0.3425 + 690000 -5566.4304 1231.9604 -6798.3908 1457.3283 21668.417 -30337.919 295.42329 287.07228 0.98982439 0.345 + 695000 -5507.4527 1265.2566 -6772.7094 1366.7935 21778.418 -30340.407 303.4077 -166.79718 1.0097878 0.3475 + 700000 -5621.3066 1277.4101 -6898.7167 1529.2337 21477.58 -30339.84 306.32209 1244.9468 1.0235434 0.35 + 705000 -5503.7028 1288.0097 -6791.7124 1430.0691 21713.551 -30338.914 308.86387 480.01323 1.0067532 0.3525 + 710000 -5483.2647 1299.6448 -6782.9095 1336.3828 21780.796 -30336.31 311.65398 -744.40786 0.9955777 0.355 + 715000 -5552.0795 1213.1147 -6765.1942 1367.5913 21780.545 -30334.566 290.90411 -182.26292 1.0179523 0.3575 + 720000 -5560.2543 1223.8905 -6784.1448 1469.5204 21612.085 -30334.718 293.48814 686.798 1.0061298 0.36 + 725000 -5352.2877 1277.7298 -6630.0175 1247.1535 21984.591 -30332.503 306.39876 -1113.3988 1.0031932 0.3625 + 730000 -5460.2162 1294.0163 -6754.2324 1328.4251 21812.669 -30338.962 310.30425 -703.10487 1.0076915 0.365 + 735000 -5484.3003 1263.9561 -6748.2564 1391.8724 21763.598 -30333.325 303.09585 358.57571 1.0159982 0.3675 + 740000 -5561.5146 1232.365 -6793.8795 1417.041 21689.319 -30337.153 295.52031 292.49943 1.0148461 0.37 + 745000 -5630.3272 1239.8819 -6870.2091 1474.7847 21577.482 -30336.129 297.32287 691.90387 1.0121157 0.3725 + 750000 -5568.5982 1252.3637 -6820.9619 1394.8143 21677.258 -30335.92 300.31599 -119.75657 1.0137126 0.375 + 755000 -5612.0848 1219.7512 -6831.8361 1443.772 21640.182 -30334.138 292.49554 419.10212 1.0092041 0.3775 + 760000 -5558.6692 1246.8503 -6805.5195 1383.8773 21739.076 -30332.761 298.99389 -360.94924 1.0057415 0.38 + 765000 -5549.313 1264.7175 -6814.0305 1361.0876 21760.857 -30334.954 303.27841 -657.15125 0.99708711 0.3825 + 770000 -5529.7814 1271.3249 -6801.1063 1515.765 21530.178 -30335.782 304.86287 1252.2865 1.0144338 0.385 + 775000 -5590.0803 1229.8806 -6819.9609 1497.7379 21574.205 -30337.017 294.92455 875.72015 1.0034086 0.3875 + 780000 -5453.0919 1304.4967 -6757.5885 1382.0095 21780.373 -30336.825 312.81744 94.669052 1.0004685 0.39 + 785000 -5534.5816 1224.2253 -6758.8069 1434.4738 21697.922 -30337.941 293.56842 992.15486 1.0337883 0.3925 + 790000 -5502.6322 1265.271 -6767.9032 1398.8135 21757.098 -30337.587 303.41116 116.52301 1.0001157 0.395 + 795000 -5493.1146 1274.5478 -6767.6624 1388.2754 21742.779 -30337.47 305.63572 140.01039 1.0236842 0.3975 + 800000 -5588.998 1225.6585 -6814.6564 1330.7321 21731.986 -30338.455 293.91209 -865.99206 1.0138727 0.4 + 805000 -5544.0626 1266.7916 -6810.8542 1362.8312 21718.168 -30338.846 303.77579 -337.37636 1.0105636 0.4025 + 810000 -5562.8317 1225.0571 -6787.8888 1387.1409 21714.016 -30338.516 293.76787 -132.30834 1.0033838 0.405 + 815000 -5450.6822 1281.7205 -6732.4027 1384.0948 21802.411 -30334.816 307.35573 457.40601 1.0295805 0.4075 + 820000 -5517.3791 1253.0874 -6770.4665 1342.5725 21767.666 -30338.348 300.48953 -677.2962 1.0045618 0.41 + 825000 -5521.6442 1253.7177 -6775.3619 1356.0902 21761.083 -30338.876 300.64068 -148.29882 1.0214802 0.4125 + 830000 -5480.2275 1296.3533 -6776.5808 1364.7623 21737.884 -30340.245 310.86468 -335.51468 1.0183724 0.415 + 835000 -5553.6513 1255.3646 -6809.0158 1422.3832 21678.078 -30336.06 301.03559 240.25343 1.0133683 0.4175 + 840000 -5487.5608 1239.6145 -6727.1753 1346.503 21835.502 -30338.024 297.25873 -317.28429 1.0034118 0.42 + 845000 -5515.8021 1255.1208 -6770.9229 1514.9877 21605.011 -30337.481 300.97715 1627.3413 1.0134437 0.4225 + 850000 -5540.0383 1285.5034 -6825.5417 1428.3295 21672.551 -30335.853 308.26287 466.86907 1.0199088 0.425 + 855000 -5597.4828 1214.2794 -6811.7622 1316.2636 21742.131 -30335.968 291.18339 -850.89507 1.0272315 0.4275 + 860000 -5442.4541 1274.0556 -6716.5098 1338.0091 21820.114 -30337.949 305.51769 -629.60481 1.0007525 0.43 + 865000 -5532.6042 1268.7742 -6801.3783 1434.2572 21700.789 -30338.172 304.25121 492.79858 1.0070617 0.4325 + 870000 -5512.1578 1258.0877 -6770.2455 1454.9091 21687.255 -30336.232 301.68859 683.83531 1.0116145 0.435 + 875000 -5446.697 1264.9331 -6711.6301 1418.0271 21789.48 -30337.979 303.33012 746.51851 1.0077359 0.4375 + 880000 -5570.2405 1227.4141 -6797.6546 1442.7982 21657.832 -30339.523 294.3331 547.05287 1.0204637 0.44 + 885000 -5499.5832 1300.6877 -6800.2708 1467.889 21621.607 -30336.081 311.90404 1203.3728 1.0232175 0.4425 + 890000 -5519.9378 1221.4368 -6741.3746 1334.4295 21836.44 -30334.429 292.89974 -571.0912 1.0183523 0.445 + 895000 -5548.714 1235.0959 -6783.81 1335.7422 21783.359 -30339.641 296.17519 -1231.4103 0.97545271 0.4475 + 900000 -5509.0007 1255.7581 -6764.7588 1387.4098 21730.996 -30336.325 301.12995 100.61482 1.0157715 0.45 + 905000 -5502.2972 1203.2327 -6705.5298 1412.9802 21806.928 -30335.359 288.5344 380.80789 0.99980077 0.4525 + 910000 -5485.3884 1248.0555 -6733.4439 1324.7849 21816.875 -30334.132 299.28288 -875.85577 1.0005401 0.455 + 915000 -5570.7334 1227.7102 -6798.4436 1414.6961 21676.719 -30333.844 294.40409 387.60411 1.0183984 0.4575 + 920000 -5506.8684 1250.8936 -6757.762 1403.8787 21724.014 -30338.2 299.96345 268.40619 1.0039537 0.46 + 925000 -5499.5513 1267.5801 -6767.1314 1384.0561 21746.132 -30334.877 303.96486 -87.373383 1.0135168 0.4625 + 930000 -5526.6263 1248.6532 -6775.2794 1422.5927 21727.532 -30335.327 299.42621 402.99079 1.0166018 0.465 + 935000 -5431.7671 1287.9484 -6719.7154 1372.6887 21797.491 -30339.576 308.84917 -73.380785 1.0086899 0.4675 + 940000 -5523.2347 1238.0317 -6761.2664 1472.3922 21690.931 -30338.509 296.87919 934.24131 1.0034105 0.47 + 945000 -5492.1094 1287.7201 -6779.8294 1409.131 21722.941 -30338.562 308.79442 393.63885 1.0121904 0.4725 + 950000 -5511.4769 1269.7709 -6781.2477 1399.8709 21719.432 -30338.344 304.49021 548.56247 1.0269619 0.475 + 955000 -5522.0084 1210.1211 -6732.1295 1354.1734 21827.644 -30337.824 290.18625 -328.52346 1.0035058 0.4775 + 960000 -5503.1738 1294.651 -6797.8248 1457.7604 21658.337 -30336.945 310.45646 920.37836 1.0242902 0.48 + 965000 -5574.7316 1224.5025 -6799.234 1341.877 21730.147 -30339.647 293.63489 -630.41891 1.020801 0.4825 + 970000 -5465.502 1254.5251 -6720.0271 1369.3231 21814.519 -30334.836 300.8343 412.848 1.018357 0.485 + 975000 -5429.0617 1314.1038 -6743.1655 1370.3831 21794.845 -30335.871 315.12122 -22.569108 1.0125271 0.4875 + 980000 -5509.8072 1268.7786 -6778.5858 1416.9817 21666.164 -30336.733 304.25227 605.68124 1.027029 0.49 + 985000 -5553.4756 1250.0201 -6803.4957 1399.3561 21680.727 -30336.562 299.754 -204.2377 1.0018705 0.4925 + 990000 -5509.8976 1209.5413 -6719.4389 1387.6207 21819.75 -30335.66 290.0472 71.030235 1.001716 0.495 + 995000 -5477.6146 1307.6601 -6785.2746 1376.1635 21750.915 -30337.489 313.57602 190.04886 1.0238268 0.4975 + 1000000 -5565.6634 1225.3286 -6790.992 1324.6248 21756.234 -30339.456 293.83299 -1026.1772 1.0132391 0.5 + 1005000 -5521.1329 1240.5003 -6761.6332 1339.8715 21795.557 -30333.404 297.47116 -764.80467 0.99627553 0.5025 + 1010000 -5533.5215 1234.4615 -6767.983 1467.6615 21687.246 -30339.234 296.02306 1106.3598 1.0156178 0.505 + 1015000 -5520.4449 1273.906 -6794.3509 1418.9447 21652.099 -30339.308 305.48182 508.55548 1.0227299 0.5075 + 1020000 -5553.6528 1269.5526 -6823.2054 1438.6041 21621.622 -30336.816 304.43788 356.09604 1.0009266 0.51 + 1025000 -5540.9284 1252.3914 -6793.3197 1384.0812 21720.397 -30333.925 300.32262 -137.53514 1.0150782 0.5125 + 1030000 -5512.2551 1266.8676 -6779.1227 1325.0473 21775.04 -30335.967 303.794 -785.84041 1.0128966 0.515 + 1035000 -5496.6909 1240.9291 -6737.62 1437.7445 21714.713 -30333.336 297.57398 639.06592 1.0104672 0.5175 + 1040000 -5500.8602 1277.8725 -6778.7327 1415.4978 21718.412 -30339.752 306.43298 398.08262 1.0150888 0.52 + 1045000 -5557.4463 1221.9391 -6779.3854 1415.6692 21681.767 -30337.275 293.02019 225.82498 1.0202672 0.5225 + 1050000 -5529.7555 1258.6892 -6788.4447 1422.0929 21704.679 -30338.539 301.83284 524.89602 1.0134303 0.525 + 1055000 -5518.6662 1299.6689 -6818.3351 1372.5056 21708.14 -30339.378 311.65974 -452.91496 1.0046315 0.5275 + 1060000 -5540.9771 1210.2217 -6751.1987 1373.9164 21768.116 -30336.924 290.21036 -395.70437 1.0015571 0.53 + 1065000 -5518.9029 1251.169 -6770.0719 1340.1507 21808.131 -30336.104 300.02949 -450.07574 1.0129813 0.5325 + 1070000 -5463.4785 1293.1208 -6756.5994 1332.4554 21823.853 -30339.89 310.08952 -322.88053 1.0120798 0.535 + 1075000 -5515.696 1249.2852 -6764.9812 1330.0946 21799.218 -30334.329 299.57777 -715.46839 1.0121899 0.5375 + 1080000 -5504.3342 1276.1607 -6780.4949 1300.5086 21805.677 -30337.953 306.02248 -1428.0385 0.99943418 0.54 + 1085000 -5509.2646 1255.0718 -6764.3364 1371.5075 21811.728 -30336.843 300.96538 73.910297 1.0082101 0.5425 + 1090000 -5412.182 1249.5709 -6661.753 1328.4884 21934.293 -30335.494 299.64628 -338.59758 0.99997407 0.545 + 1095000 -5473.6992 1287.1358 -6760.835 1347.4996 21804.548 -30337.264 308.65431 -369.832 1.0081179 0.5475 + 1100000 -5516.2354 1272.7455 -6788.9809 1377.2139 21731.758 -30339.241 305.20352 -33.679902 1.016951 0.55 + 1105000 -5501.6136 1255.1998 -6756.8134 1345.4796 21794.16 -30333.339 300.99608 -267.7936 1.0171513 0.5525 + 1110000 -5490.4441 1286.0141 -6776.4582 1389.8357 21706.333 -30336.453 308.38533 239.72552 1.017338 0.555 + 1115000 -5508.2867 1224.7333 -6733.02 1416.9404 21736.456 -30336.779 293.69023 411.88891 1.0061343 0.5575 + 1120000 -5536.5054 1242.9899 -6779.4953 1457.1964 21629.489 -30339.119 298.06817 856.72217 1.0160345 0.56 + 1125000 -5486.6942 1294.0843 -6780.7786 1358.1577 21782.945 -30339.37 310.32057 -202.18969 1.0178261 0.5625 + 1130000 -5436.8131 1262.4873 -6699.3004 1386.0247 21826.27 -30335.282 302.74362 322.86538 0.99696282 0.565 + 1135000 -5565.1777 1207.4654 -6772.643 1336.6359 21809.289 -30337.777 289.54939 -972.29994 0.98448542 0.5675 + 1140000 -5492.0308 1247.1935 -6739.2243 1402.2517 21754.689 -30337.029 299.07618 479.30775 1.0200599 0.57 + 1145000 -5594.1678 1257.8479 -6852.0157 1434.3633 21605.553 -30337.373 301.6311 164.16272 1.0163196 0.5725 + 1150000 -5595.7162 1204.3256 -6800.0418 1360.4415 21756.387 -30337.378 288.79648 -673.65939 0.99447246 0.575 + 1155000 -5560.4788 1255.9123 -6816.3911 1370.1918 21717.925 -30336.419 301.16694 -321.48858 1.0102569 0.5775 + 1160000 -5543.9624 1256.0273 -6799.9897 1361.3625 21729.13 -30336.145 301.19451 -500.9834 1.0075609 0.58 + 1165000 -5542.3677 1259.1894 -6801.5571 1392.601 21712.586 -30336.732 301.95279 269.67035 1.028619 0.5825 + 1170000 -5507.6711 1265.0503 -6772.7214 1392.4757 21734.407 -30337.47 303.35823 -1.1384906 1.0009475 0.585 + 1175000 -5616.4108 1190.6661 -6807.0769 1485.5233 21602.259 -30336.566 285.52095 797.79093 1.004332 0.5875 + 1180000 -5452.7462 1270.5601 -6723.3063 1380.6407 21772.422 -30339.576 304.67948 224.518 1.0138518 0.59 + 1185000 -5551.9378 1288.6384 -6840.5761 1409.257 21663.998 -30334.605 309.01463 201.047 1.0216943 0.5925 + 1190000 -5401.6296 1286.2182 -6687.8478 1413.7575 21821.846 -30335.244 308.43426 792.08684 1.0158526 0.595 + 1195000 -5595.7083 1241.8942 -6837.6025 1344.7629 21737.232 -30336.445 297.80541 -775.15064 1.0117893 0.5975 + 1200000 -5514.8805 1263.3084 -6778.1889 1427.3516 21696.321 -30339.4 302.94052 485.34384 1.0109195 0.6 + 1205000 -5540.2706 1219.5591 -6759.8297 1339.7958 21822.653 -30337.629 292.44947 -429.03962 1.0085747 0.6025 + 1210000 -5490.2376 1291.785 -6782.0226 1468.9881 21640.937 -30335.388 309.76919 1374.7882 1.0248876 0.605 + 1215000 -5556.2871 1242.1303 -6798.4174 1355.9924 21726.062 -30334.653 297.86202 -392.05916 1.0151509 0.6075 + 1220000 -5511.0713 1246.1099 -6757.1812 1438.275 21704.752 -30334.819 298.81633 866.53777 1.0152101 0.61 + 1225000 -5425.1538 1241.4366 -6666.5904 1407.8381 21815.403 -30338.21 297.69567 781.0192 1.0077519 0.6125 + 1230000 -5566.9066 1231.8806 -6798.7872 1414.1772 21716.877 -30339.764 295.40416 581.48996 1.022715 0.615 + 1235000 -5477.752 1240.0569 -6717.8089 1368.5535 21803.025 -30337.639 297.36483 257.90729 1.0209426 0.6175 + 1240000 -5518.2359 1241.8471 -6760.083 1357.7485 21769.434 -30333.502 297.79412 -403.71101 1.0002981 0.62 + 1245000 -5505.2401 1253.7734 -6759.0135 1356.1495 21781.172 -30338.614 300.65403 -238.83176 1.019027 0.6225 + 1250000 -5481.0309 1269.2078 -6750.2387 1422.4459 21736.094 -30334.948 304.35519 389.35305 0.99684561 0.625 + 1255000 -5591.0366 1268.5978 -6859.6344 1440.4456 21599.606 -30339.235 304.20892 479.16928 1.0279113 0.6275 + 1260000 -5458.4807 1274.8262 -6733.3069 1350.7336 21797.841 -30336.376 305.70248 -272.75408 1.0075811 0.63 + 1265000 -5575.7251 1215.4896 -6791.2147 1393.8174 21695.141 -30338.69 291.47361 -440.69662 0.9946576 0.6325 + 1270000 -5506.6538 1247.2498 -6753.9037 1426.4163 21735.083 -30339.604 299.08968 638.29292 1.006986 0.635 + 1275000 -5555.7487 1260.8981 -6816.6467 1402.9042 21687.407 -30338.211 302.36252 -37.621691 1.004614 0.6375 + 1280000 -5523.0115 1281.4484 -6804.46 1410.3618 21683.821 -30338.245 307.29049 74.776331 1.0067829 0.64 + 1285000 -5495.0391 1269.3944 -6764.4336 1438.0896 21688.271 -30338.328 304.39995 589.55661 1.0114896 0.6425 + 1290000 -5540.9271 1240.5911 -6781.5182 1328.7135 21747.08 -30332.671 297.49292 -1009.5288 0.99417236 0.645 + 1295000 -5523.5629 1221.2216 -6744.7846 1386.738 21787.399 -30341.525 292.84814 152.73721 1.020105 0.6475 + 1300000 -5551.0533 1209.1809 -6760.2342 1283.6253 21834.417 -30338.537 289.96078 -1463.7036 0.99836441 0.65 + 1305000 -5482.2712 1243.9505 -6726.2216 1443.3829 21700.095 -30334.574 298.2985 937.35283 1.0133398 0.6525 + 1310000 -5497.8833 1275.4343 -6773.3175 1331.5717 21786.443 -30338.584 305.84829 -568.49832 1.0153372 0.655 + 1315000 -5553.0213 1263.5103 -6816.5316 1328.3328 21741.571 -30339.561 302.98894 -899.43861 1.0107628 0.6575 + 1320000 -5497.4183 1275.5116 -6772.93 1334.6287 21757.028 -30335.101 305.86685 -750.04561 1.0127122 0.66 + 1325000 -5509.5201 1280.9643 -6790.4845 1417.3333 21664.023 -30335.168 307.1744 -157.6594 0.99319825 0.6625 + 1330000 -5617.2597 1224.5019 -6841.7616 1443.6235 21625.123 -30339.1 293.63475 717.99095 1.0298596 0.665 + 1335000 -5592.8391 1248.1956 -6841.0347 1387.7561 21696.521 -30337.925 299.31648 -247.91846 1.0135438 0.6675 + 1340000 -5460.2548 1288.4813 -6748.7361 1360.0244 21811.949 -30338.096 308.97696 -153.69598 1.015044 0.67 + 1345000 -5489.1678 1281.692 -6770.8598 1431.0453 21663.894 -30338.844 307.3489 669.39702 1.0172647 0.6725 + 1350000 -5529.2523 1270.9326 -6800.1849 1363.1373 21733.208 -30339.456 304.7688 -597.14779 1.0042186 0.675 + 1355000 -5534.9568 1265.7444 -6800.7012 1409.764 21689.237 -30334.88 303.52466 183.99575 1.0031237 0.6775 + 1360000 -5570.4522 1226.0935 -6796.5457 1409.4114 21697.645 -30337.424 294.01641 486.08744 1.0257807 0.68 + 1365000 -5491.4382 1268.1819 -6759.6201 1338.2699 21793.974 -30334.116 304.10918 -642.86253 1.0062663 0.6825 + 1370000 -5490.238 1223.206 -6713.4439 1291.9792 21860.632 -30336.796 293.32398 -1031.6635 0.99972319 0.685 + 1375000 -5533.2447 1274.0075 -6807.2521 1352.1474 21702.877 -30337.44 305.50615 -487.37416 1.0147896 0.6875 + 1380000 -5473.8114 1276.8204 -6750.6318 1407.4579 21700.787 -30335.345 306.18068 249.06913 1.0069428 0.69 + 1385000 -5543.9666 1262.8767 -6806.8434 1420.4958 21674.581 -30335.208 302.83701 250.9923 1.0125827 0.6925 + 1390000 -5502.5865 1244.5168 -6747.1034 1404.518 21737.859 -30331.637 298.43431 272.2078 1.0080372 0.695 + 1395000 -5485.4502 1277.3217 -6762.7719 1328.4992 21816.677 -30335.548 306.3009 -482.56908 1.0234689 0.6975 + 1400000 -5588.9107 1238.0105 -6826.9212 1441.2329 21623.183 -30339.033 296.8741 219.31408 1.0059101 0.7 + 1405000 -5550.2667 1211.9058 -6762.1725 1365.9796 21743.769 -30331.304 290.61421 -173.3426 1.0210828 0.7025 + 1410000 -5486.1246 1271.3405 -6757.4651 1425.326 21739.7 -30337.946 304.86661 874.21456 1.0139115 0.705 + 1415000 -5537.3934 1269.7516 -6807.1451 1449.2446 21658.871 -30335.68 304.4856 569.30023 1.0046679 0.7075 + 1420000 -5560.5616 1245.2367 -6805.7983 1402.5678 21708.73 -30335.385 298.60694 -104.6055 1.0091745 0.71 + 1425000 -5528.1555 1294.6881 -6822.8437 1514.7057 21531.479 -30337.473 310.46536 1522.6101 1.0126646 0.7125 + 1430000 -5497.5001 1287.4796 -6784.9798 1392.3932 21699.849 -30339.103 308.73677 61.912759 1.00971 0.715 + 1435000 -5510.8749 1237.5563 -6748.4312 1358.3772 21788.933 -30340.288 296.76518 -306.88924 1.0018599 0.7175 + 1440000 -5499.3556 1270.3888 -6769.7443 1370.7823 21727.629 -30339.175 304.63839 -53.340447 1.0200557 0.72 + 1445000 -5546.7228 1232.4352 -6779.1579 1373.6312 21729.152 -30336.67 295.53714 -213.61291 1.0138642 0.7225 + 1450000 -5449.6319 1267.9184 -6717.5503 1369.4309 21793.674 -30334.448 304.04599 184.06979 1.0177429 0.725 + 1455000 -5605.5581 1205.3164 -6810.8745 1379.1631 21695.098 -30340.184 289.03407 -396.85833 1.0020238 0.7275 + 1460000 -5624.6928 1228.0852 -6852.778 1404.3538 21660.898 -30340.227 294.49403 -103.24316 1.0176862 0.73 + 1465000 -5517.9353 1237.1764 -6755.1117 1376.5763 21773.353 -30337.713 296.67409 -143.30329 1.0091248 0.7325 + 1470000 -5531.193 1192.152 -6723.345 1469.0677 21691.379 -30335.647 285.87727 1394.3946 1.0169307 0.735 + 1475000 -5469.5125 1278.1335 -6747.6459 1374.8894 21790.652 -30336.291 306.49556 -320.76133 0.99923276 0.7375 + 1480000 -5569.7546 1225.3414 -6795.096 1369.5149 21704.23 -30339.752 293.83607 -295.5674 1.0086 0.74 + 1485000 -5570.7676 1229.0249 -6799.7924 1358.7067 21754.36 -30339.644 294.71935 -446.59245 1.0146734 0.7425 + 1490000 -5578.5093 1213.6038 -6792.1131 1361.3024 21746.493 -30339.324 291.0214 -279.50424 1.0188708 0.745 + 1495000 -5574.1959 1286.4102 -6860.6061 1391.1258 21669.913 -30338.466 308.48032 -404.60846 1.0074475 0.7475 + 1500000 -5499.0758 1257.807 -6756.8828 1293.9978 21854.746 -30334.433 301.62128 -995.33111 1.0061023 0.75 + 1505000 -5555.8386 1224.7451 -6780.5837 1398.2158 21721.994 -30333.807 293.69307 -108.77374 1.0017368 0.7525 + 1510000 -5516.6813 1232.9495 -6749.6308 1375.2615 21790.724 -30334.453 295.66048 -254.2256 0.99292399 0.755 + 1515000 -5506.9453 1232.1963 -6739.1416 1398.9195 21765.1 -30337.448 295.47987 280.5709 1.0059619 0.7575 + 1520000 -5572.2466 1203.9157 -6776.1623 1358.6224 21732.12 -30335.917 288.69819 -423.60315 1.013085 0.76 + 1525000 -5479.3423 1275.6547 -6754.997 1348.5224 21788.359 -30339.133 305.90116 -332.75562 1.0119609 0.7625 + 1530000 -5518.4409 1217.0162 -6735.4571 1369.8978 21790.403 -30338.036 291.83967 -211.5446 0.99823021 0.765 + 1535000 -5517.1315 1253.9634 -6771.0948 1424.4081 21690.09 -30337.668 300.69959 341.83304 1.0041481 0.7675 + 1540000 -5528.0888 1233.7041 -6761.7929 1405.9083 21710.981 -30336.39 295.84144 -32.35231 0.99593709 0.77 + 1545000 -5539.0412 1238.631 -6777.6723 1311.2439 21794.343 -30337.582 297.0229 -1311.3664 0.9961313 0.7725 + 1550000 -5595.0349 1240.5672 -6835.602 1441.0277 21670.814 -30338.678 297.48719 536.63002 1.015083 0.775 + 1555000 -5560.1434 1270.6028 -6830.7462 1443.7953 21637.472 -30333.488 304.68972 470.00368 1.0055598 0.7775 + 1560000 -5496.2344 1234.9637 -6731.1981 1330.1568 21821.108 -30335.645 296.14347 -757.48104 0.99041343 0.78 + 1565000 -5556.0238 1248.445 -6804.4688 1397.6434 21692.421 -30338.719 299.37629 -10.691405 1.0182037 0.7825 + 1570000 -5552.3079 1243.8513 -6796.1592 1397.511 21703.363 -30336.571 298.27472 32.033113 1.0089905 0.785 + 1575000 -5468.3273 1283.6822 -6752.0094 1361.1407 21787.216 -30336.574 307.82614 -164.38813 1.0083064 0.7875 + 1580000 -5488.0146 1232.0833 -6720.0979 1320.826 21829.873 -30335.918 295.45277 -417.97233 1.018807 0.79 + 1585000 -5535.136 1231.2656 -6766.4015 1440.3397 21691.223 -30338.944 295.25667 686.24285 1.0137355 0.7925 + 1590000 -5507.2274 1257.8264 -6765.0537 1388.9839 21718.772 -30335.685 301.62593 -138.69824 1.003965 0.795 + 1595000 -5493.0812 1223.4504 -6716.5316 1396.9927 21762.807 -30334.042 293.38261 250.04963 1.0085422 0.7975 + 1600000 -5542.7694 1256.563 -6799.3324 1383.1567 21709.773 -30338.003 301.32298 -51.465013 1.0131684 0.8 + 1605000 -5453.3692 1285.3523 -6738.7215 1362.5454 21803.391 -30335.547 308.22664 -321.46323 1.0029343 0.8025 + 1610000 -5503.9352 1259.2527 -6763.188 1421.5945 21696.436 -30334.944 301.96798 625.60637 1.0255178 0.805 + 1615000 -5476.5761 1246.0171 -6722.5932 1347.6694 21806.697 -30336.05 298.79408 -91.329161 1.0116498 0.8075 + 1620000 -5507.707 1237.031 -6744.738 1371.7693 21775.289 -30334.991 296.63921 -213.66643 1.0063229 0.81 + 1625000 -5456.168 1228.6159 -6684.7839 1372.1719 21806.244 -30336.186 294.62128 293.33707 1.0094967 0.8125 + 1630000 -5650.3002 1225.7447 -6876.0449 1465.0474 21543.498 -30336.757 293.93277 417.30626 1.0158098 0.815 + 1635000 -5450.49 1275.0111 -6725.5011 1481.2268 21684.726 -30337.105 305.74683 1426.6931 1.0094353 0.8175 + 1640000 -5617.7996 1227.1912 -6844.9908 1412.9007 21647.959 -30339.808 294.27965 -64.670241 1.0184979 0.82 + 1645000 -5479.4668 1286.8935 -6766.3603 1430.311 21716.402 -30338.791 308.59622 613.1652 1.002358 0.8225 + 1650000 -5500.2183 1280.3487 -6780.567 1385.4426 21748.104 -30337.348 307.02677 -65.179279 1.0098634 0.825 + 1655000 -5560.2962 1223.2516 -6783.5478 1311.1561 21783.296 -30335.262 293.33493 -1444.809 0.98862533 0.8275 + 1660000 -5560.1654 1221.3779 -6781.5433 1384.641 21766.196 -30337.4 292.88562 -30.809802 1.0055892 0.83 + 1665000 -5459.1135 1261.4358 -6720.5493 1334.6877 21855.177 -30336.221 302.49147 -691.93836 0.98631443 0.8325 + 1670000 -5426.4876 1277.2456 -6703.7331 1301.4361 21883.289 -30338.807 306.28264 -769.6473 1.0029395 0.835 + 1675000 -5560.3658 1228.1155 -6788.4813 1421.5285 21710.575 -30339.538 294.50128 134.14929 0.99950546 0.8375 + 1680000 -5533.0079 1239.2308 -6772.2386 1466.9684 21661.29 -30336.838 297.16672 1000.7926 1.0093582 0.84 + 1685000 -5532.7803 1282.6992 -6815.4795 1387.5896 21673.135 -30336.269 307.59043 -293.90054 1.0036548 0.8425 + 1690000 -5533.8573 1242.0398 -6775.8971 1295.4859 21813.204 -30335.918 297.84032 -1072.3783 1.0103958 0.845 + 1695000 -5569.1131 1236.3099 -6805.423 1391.4947 21734.459 -30337.665 296.46629 -42.852437 1.0020314 0.8475 + 1700000 -5514.6926 1236.9596 -6751.6522 1297.2083 21861.079 -30337.966 296.6221 -1035.3111 1.0148487 0.85 + 1705000 -5481.7854 1264.9348 -6746.7202 1394.4051 21765.618 -30334.989 303.33053 68.230386 0.99805471 0.8525 + 1710000 -5628.7652 1218.5732 -6847.3385 1424.8975 21629.726 -30339.018 292.21306 -18.822494 1.0061359 0.855 + 1715000 -5535.4768 1269.0696 -6804.5464 1310.4991 21778.662 -30338.107 304.32206 -1128.3429 1.0076215 0.8575 + 1720000 -5612.6083 1193.1968 -6805.8051 1383.4907 21742.438 -30339.657 286.12782 -186.2314 1.0077134 0.86 + 1725000 -5483.1527 1278.2192 -6761.3719 1414.7937 21718.362 -30332.789 306.51613 275.89425 0.99757317 0.8625 + 1730000 -5507.2292 1239.3969 -6746.626 1349.4125 21760.595 -30336.33 297.20655 -160.96278 1.0175868 0.865 + 1735000 -5471.6725 1274.6169 -6746.2893 1353.4867 21778.13 -30334.321 305.65228 -455.73375 0.99473861 0.8675 + 1740000 -5457.1173 1292.0777 -6749.195 1354.1425 21798.549 -30337.698 309.83938 105.28166 1.013856 0.87 + 1745000 -5526.7699 1217.2299 -6743.9998 1374.1349 21775.087 -30335.943 291.89093 21.249651 1.0097813 0.8725 + 1750000 -5558.9441 1238.0218 -6796.966 1398.6419 21742.403 -30337.165 296.87681 258.79723 1.0189989 0.875 + 1755000 -5510.581 1278.8181 -6789.3991 1388.2658 21710.876 -30337.981 306.65974 475.10366 1.0315721 0.8775 + 1760000 -5530.9869 1295.8685 -6826.8554 1479.5118 21573.341 -30338.944 310.74841 748.89788 1.0132734 0.88 + 1765000 -5533.3839 1317.0507 -6850.4347 1435.6808 21644.207 -30333.746 315.8279 594.0529 1.0192713 0.8825 + 1770000 -5465.8084 1240.5045 -6706.3129 1292.0385 21900.378 -30339.129 297.47215 -1025.2013 1.0024888 0.885 + 1775000 -5599.0359 1221.6792 -6820.7151 1462.1624 21619.581 -30339.425 292.95787 626.3862 1.0091902 0.8875 + 1780000 -5538.0385 1264.6729 -6802.7114 1372.7663 21730.471 -30338.894 303.26773 -192.55416 1.0065058 0.89 + 1785000 -5428.1822 1333.9892 -6762.1715 1389.1017 21728.7 -30336.27 319.88974 104.29655 1.0139352 0.8925 + 1790000 -5508.9545 1287.0739 -6796.0284 1445.185 21652.743 -30336.902 308.63947 391.87637 1.0000323 0.895 + 1795000 -5523.6516 1235.1646 -6758.8162 1418.1244 21721.296 -30337.575 296.19166 666.61502 1.0229146 0.8975 + 1800000 -5572.6377 1214.1205 -6786.7582 1392.4223 21722.483 -30335.66 291.14528 -97.658277 1.0013673 0.9 + 1805000 -5546.2419 1219.6808 -6765.9227 1366.9631 21762.107 -30337.944 292.47866 -57.351578 1.0188349 0.9025 + 1810000 -5570.7715 1247.1279 -6817.8994 1386.5773 21707.994 -30338.467 299.06045 128.40982 1.018309 0.905 + 1815000 -5527.1768 1236.8139 -6763.9907 1370.0226 21756.947 -30337.433 296.58716 -58.429002 1.0156077 0.9075 + 1820000 -5392.8702 1296.598 -6689.4682 1422.7107 21835.725 -30339.535 310.92334 1096.9698 1.0098001 0.91 + 1825000 -5403.1407 1273.2353 -6676.3761 1398.7473 21821.771 -30338.272 305.32099 575.09164 1.009625 0.9125 + 1830000 -5533.275 1223.962 -6757.237 1302.4782 21879.688 -30335.303 293.50528 -794.93635 1.0164392 0.915 + 1835000 -5537.9594 1230.3716 -6768.331 1359.2088 21780.105 -30337.746 295.04229 -301.96771 1.012533 0.9175 + 1840000 -5496.5607 1259.4196 -6755.9803 1392.4138 21810.816 -30335.129 302.00799 308.79809 1.0087261 0.92 + 1845000 -5516.8276 1233.6195 -6750.4471 1356.0628 21770.257 -30336.985 295.82114 -563.13925 0.99534053 0.9225 + 1850000 -5551.2862 1216.2779 -6767.5641 1315.4137 21779.186 -30338.525 291.66263 -1234.4928 0.9977088 0.925 + 1855000 -5514.4008 1278.8525 -6793.2533 1477.9866 21661.793 -30338.935 306.668 1192.9778 1.0126178 0.9275 + 1860000 -5445.2459 1262.0113 -6707.2573 1295.8277 21920.956 -30333.967 302.62948 -1093.2105 0.98876382 0.93 + 1865000 -5479.9894 1263.9698 -6743.9592 1342.5201 21812.688 -30335.4 303.09913 -141.70262 1.018338 0.9325 + 1870000 -5526.4829 1205.3702 -6731.8531 1376.0037 21811.821 -30332.663 289.04698 186.81383 1.0172778 0.935 + 1875000 -5566.9793 1221.657 -6788.6363 1352.419 21753.184 -30332.134 292.95254 -421.96475 1.0091425 0.9375 + 1880000 -5497.1676 1282.0374 -6779.205 1398.213 21722.715 -30339.838 307.43173 37.827778 1.0039977 0.94 + 1885000 -5557.4404 1233.9174 -6791.3578 1449.936 21642.09 -30339.056 295.89259 883.49759 1.0130214 0.9425 + 1890000 -5506.733 1295.1267 -6801.8597 1297.6178 21817.614 -30338.332 310.57052 -1073.8778 1.0180788 0.945 + 1895000 -5533.7516 1257.448 -6791.1996 1345.2106 21764.458 -30338.571 301.53521 -526.55649 1.0121615 0.9475 + 1900000 -5541.0051 1258.9341 -6799.9391 1381.6398 21733.321 -30338.593 301.89155 -252.04317 1.008764 0.95 + 1905000 -5609.2146 1182.6597 -6791.8743 1350.4308 21757.084 -30338.743 283.60101 -97.157492 1.0309631 0.9525 + 1910000 -5571.9982 1254.2855 -6826.2837 1389.361 21684.388 -30336.514 300.77683 -350.37238 1.0002085 0.955 + 1915000 -5504.046 1277.8838 -6781.9299 1392.3191 21732.799 -30339.549 306.43569 295.58866 1.0234965 0.9575 + 1920000 -5500.6241 1311.1129 -6811.737 1384.3734 21674.326 -30341.238 314.40401 84.788637 1.0209565 0.96 + 1925000 -5529.7992 1229.6795 -6759.4787 1408.9276 21763.324 -30339.966 294.87632 215.06361 1.0061547 0.9625 + 1930000 -5444.6302 1282.6892 -6727.3194 1334.5948 21864.664 -30333.059 307.58803 -76.344233 1.0195039 0.965 + 1935000 -5506.5314 1250.2755 -6756.8069 1346.0266 21804.791 -30335.384 299.81524 -400.10311 1.009444 0.9675 + 1940000 -5595.6735 1257.0187 -6852.6922 1445.275 21568.516 -30337.62 301.43225 543.8544 1.0288045 0.97 + 1945000 -5555.7495 1224.7877 -6780.5372 1333.6985 21795.754 -30338.445 293.70329 -560.62987 1.0145559 0.9725 + 1950000 -5481.8697 1239.3038 -6721.1735 1436.353 21745.545 -30335.269 297.18424 1090.0854 1.024844 0.975 + 1955000 -5586.7772 1263.8812 -6850.6584 1479.012 21594.084 -30335.219 303.07787 1031.8973 1.018222 0.9775 + 1960000 -5495.753 1254.4979 -6750.2508 1386.5525 21763.46 -30336.981 300.82776 72.271681 1.0073177 0.98 + 1965000 -5532.6074 1226.085 -6758.6924 1373.4537 21763.717 -30335.48 294.01437 49.27447 1.0207726 0.9825 + 1970000 -5592.2371 1226.5746 -6818.8117 1393.679 21701.959 -30338.233 294.13177 150.06013 1.0325882 0.985 + 1975000 -5564.1305 1237.5805 -6801.711 1398.2423 21736.76 -30338.459 296.77099 278.42773 1.0180523 0.9875 + 1980000 -5482.7493 1267.38 -6750.1292 1354.4265 21816.978 -30337.136 303.91688 -264.92549 1.0116111 0.99 + 1985000 -5486.0145 1244.3436 -6730.358 1338.1288 21844.074 -30336.86 298.39277 76.865939 1.0287137 0.9925 + 1990000 -5518.5592 1251.3748 -6769.9341 1251.2427 21885.87 -30337.966 300.07886 -1536.5189 1.0130877 0.995 + 1995000 -5487.626 1271.8163 -6759.4423 1356.5853 21769.656 -30337.831 304.9807 -374.70018 1.0071102 0.9975 + 2000000 -5510.6051 1219.5403 -6730.1454 1414.2091 21757.343 -30335.348 292.44495 576.54022 1.0102751 1 +Loop time of 13908.6 on 12 procs for 2000000 steps with 1800 atoms + +Performance: 12.424 ns/day, 1.932 hours/ns, 143.796 timesteps/s +95.4% CPU use with 12 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 7652.9 | 8199.2 | 8886.6 | 354.2 | 58.95 +Bond | 25.21 | 26.012 | 27.096 | 14.0 | 0.19 +Kspace | 2159 | 2762 | 3232.9 | 533.0 | 19.86 +Neigh | 571.65 | 572.66 | 573.54 | 2.5 | 4.12 +Comm | 806.99 | 869.79 | 940.86 | 190.9 | 6.25 +Output | 0.032536 | 0.034245 | 0.041095 | 1.2 | 0.00 +Modify | 959.22 | 1140.3 | 1306.8 | 448.8 | 8.20 +Other | | 338.6 | | | 2.43 + +Nlocal: 150.000 ave 159 max 140 min +Histogram: 1 0 2 2 0 2 2 1 0 2 +Nghost: 6121.33 ave 6236 max 6003 min +Histogram: 2 0 0 3 0 2 2 2 0 1 +Neighs: 87038.7 ave 96826 max 77686 min +Histogram: 2 0 1 3 0 1 2 1 1 1 + +Total # of neighbors = 1044464 +Ave neighs/atom = 580.25778 +Ave special neighs/atom = 2.0000000 +Neighbor list builds = 89191 +Dangerous builds = 0 +Total wall time: 4:02:50 diff --git a/examples/USER/fep/SPCEhyd/fep01/fep01-q.fep b/examples/USER/fep/SPCEhyd/fep01/fep01-q.fep new file mode 100644 index 0000000000..553c231860 --- /dev/null +++ b/examples/USER/fep/SPCEhyd/fep01/fep01-q.fep @@ -0,0 +1,22 @@ +# Time-averaged data for fix FEP +# TimeStep c_FEP[1] c_FEP[2] c_FEP[3] +100000 -0.00130466 18193.2 17742.3 +200000 -0.027526 18951.4 17742.7 +300000 -0.0459414 19629.2 17752.3 +400000 -0.0857092 21019.8 17772.6 +500000 -0.0982497 21457.5 17731.3 +600000 -0.137334 23057.4 17808.5 +700000 -0.170804 24395.7 17801 +800000 -0.223064 26797.3 17738.8 +900000 -0.261826 28800 17791.7 +1000000 -0.286374 30143.5 17783.4 +1100000 -0.349306 33728.2 17757.7 +1200000 -0.408164 37435.9 17787.7 +1300000 -0.496871 43896.5 17766.2 +1400000 -0.602917 53115.7 17749.9 +1500000 -0.72768 65474.3 17765 +1600000 -0.793802 73873.7 17785.7 +1700000 -0.912421 92487.7 17754.8 +1800000 -1.02249 109518 17762.7 +1900000 -1.12599 130115 17752.5 +2000000 -1.2899 174891 17768.3 diff --git a/examples/USER/fep/SPCEhyd/fep01/fep01-q.out b/examples/USER/fep/SPCEhyd/fep01/fep01-q.out new file mode 100644 index 0000000000..4d597f88b3 --- /dev/null +++ b/examples/USER/fep/SPCEhyd/fep01/fep01-q.out @@ -0,0 +1,561 @@ +LAMMPS (29 Oct 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (src/lammps/src/comm.cpp:94) + using 1 OpenMP thread(s) per MPI task +Reading data file ... + orthogonal box = (0.0000000 0.0000000 0.0000000) to (29.204526 29.204526 29.204526) + 2 by 2 by 3 MPI processor grid + reading atoms ... + 1800 atoms + scanning bonds ... + 1 = max bonds/atom + scanning angles ... + 1 = max angles/atom + reading bonds ... + 1200 bonds + reading angles ... + 600 angles +Finding 1-2 1-3 1-4 neighbors ... + special bond factors lj: 0 0 0.5 + special bond factors coul: 0 0 0.5 + 2 = max # of 1-2 neighbors + 1 = max # of 1-3 neighbors + 1 = max # of 1-4 neighbors + 2 = max # of special neighbors + special bonds CPU = 0.002 seconds + read_data CPU = 0.019 seconds +Finding SHAKE clusters ... + 0 = # of size 2 clusters + 600 = # of size 3 clusters + 0 = # of size 4 clusters + 0 = # of frozen angles + find clusters CPU = 0.001 seconds +Setting atom values ... + 3 settings made for charge +PPPM initialization ... + using 12-bit tables for long-range coulomb (src/lammps/src/kspace.cpp:339) + G vector (1/distance) = 0.25066829 + grid = 20 20 20 + stencil order = 5 + estimated absolute RMS force accuracy = 0.0030719151 + estimated relative force accuracy = 9.250981e-06 + using double precision FFTW3 + 3d grid and FFT values/proc = 3757 800 +Neighbor list info ... + update every 1 steps, delay 10 steps, check yes + max neighbors/atom: 2000, page size: 100000 + master list distance cutoff = 14 + ghost atom cutoff = 14 + binsize = 7, bins = 5 5 5 + 1 neighbor lists, perpetual/occasional/extra = 1 0 0 + (1) pair lj/cut/coul/long, perpetual + attributes: half, newton on + pair build: half/bin/newton + stencil: half/bin/3d/newton + bin: standard +Setting up Verlet run ... + Unit style : real + Current step : 0 + Time step : 1 +Per MPI rank memory allocation (min/avg/max) = 8.214 | 8.226 | 8.244 Mbytes +Step CPU TotEng KinEng PotEng E_vdwl E_coul E_long Temp Press Volume Density + 0 0 1344.0739 1251.046 93.027896 53.823145 30049.554 -30010.349 300 10284.878 24908.667 0.72058313 + 5000 26.813 -5450.5713 1282.0681 -6732.6394 1325.6489 21834.91 -30297.994 307.43909 -699.90679 18234.717 0.98431826 + 10000 55.604067 -5588.263 1217.9162 -6806.1792 1414.4085 21633.044 -30301.477 292.05551 23.383255 17964.154 0.99914336 + 15000 85.625894 -5502.4932 1306.1963 -6808.6896 1448.9698 21589.863 -30300.506 313.22502 674.44388 17764.828 1.010354 + 20000 119.30994 -5551.8753 1255.7906 -6807.6659 1438.0508 21600.534 -30296.631 301.13775 626.57778 17521.241 1.0244003 + 25000 153.36332 -5513.3214 1262.2974 -6775.6188 1448.1692 21624.738 -30300.649 302.69809 1082.1686 17539.064 1.0233594 + 30000 185.73754 -5517.07 1232.0851 -6749.1552 1351.1278 21743.778 -30297.746 295.45321 -345.74832 17641.782 1.0174009 + 35000 219.88466 -5461.9885 1261.1109 -6723.0993 1320.5321 21811.17 -30298.154 302.41356 -633.99545 17793.771 1.0087106 + 40000 250.59178 -5478.4688 1259.9078 -6738.3767 1361.0965 21772.195 -30298.632 302.12507 251.40827 17555.015 1.0224295 + 45000 284.497 -5586.9018 1253.0148 -6839.9166 1368.2811 21661.619 -30296.221 300.47212 -485.67054 17692.537 1.0144823 + 50000 318.07759 -5566.6473 1284.4686 -6851.116 1421.3632 21574.213 -30298.299 308.01473 123.81461 17666.827 1.0159586 + 55000 342.2475 -5571.8668 1241.516 -6813.3827 1320.1575 21745.617 -30297.514 297.71471 -757.9971 17553.891 1.0224949 + 60000 369.92714 -5405.9904 1244.2976 -6650.288 1321.6833 21864.494 -30297.613 298.38174 -252.7118 17729.038 1.0123936 + 65000 400.90471 -5536.8199 1222.3639 -6759.1838 1335.2142 21756.337 -30299.402 293.12205 -425.71516 17495.348 1.0259165 + 70000 421.35603 -5552.3284 1260.2509 -6812.5793 1340.624 21697.664 -30296.839 302.20732 -845.04229 17882.237 1.0037204 + 75000 441.98615 -5494.066 1286.3299 -6780.3959 1404.289 21683.326 -30293.835 308.46107 382.75947 17905.291 1.002428 + 80000 462.4957 -5550.6324 1269.8456 -6820.4779 1356.8274 21698.112 -30298.489 304.50813 -534.46185 17874.446 1.0041579 + 85000 483.079 -5656.6158 1200.4648 -6857.0807 1457.2846 21562.254 -30300.6 287.87068 686.28181 17521.518 1.0243841 + 90000 510.58296 -5480.2188 1242.2417 -6722.4605 1325.1999 21772.005 -30296.2 297.88874 -353.59603 17662.908 1.016184 + 95000 531.29699 -5537.8929 1265.7042 -6803.5971 1376.1449 21683.369 -30297.657 303.51504 -263.8525 17673.753 1.0155605 + 100000 563.33023 -5538.2683 1254.8684 -6793.1367 1343.4731 21751.843 -30299.503 300.91662 -623.47098 17857.853 1.0050909 +Loop time of 563.33 on 12 procs for 100000 steps with 1800 atoms + +Performance: 15.337 ns/day, 1.565 hours/ns, 177.516 timesteps/s +94.5% CPU use with 12 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 284.09 | 305.4 | 330.52 | 90.7 | 54.21 +Bond | 1.0744 | 1.1159 | 1.1633 | 3.1 | 0.20 +Kspace | 101.59 | 125.93 | 147.73 | 141.3 | 22.35 +Neigh | 22.44 | 22.466 | 22.486 | 0.3 | 3.99 +Comm | 39.278 | 41.789 | 46.639 | 43.5 | 7.42 +Output | 0.0012299 | 0.0012852 | 0.0017706 | 0.4 | 0.00 +Modify | 42.543 | 55.366 | 62.194 | 102.3 | 9.83 +Other | | 11.26 | | | 2.00 + +Nlocal: 150.000 ave 163 max 133 min +Histogram: 1 1 0 0 1 3 3 0 1 2 +Nghost: 6108.25 ave 6187 max 6048 min +Histogram: 3 0 2 1 1 1 1 1 1 1 +Neighs: 86944.1 ave 96258 max 75267 min +Histogram: 2 0 1 0 1 0 3 3 1 1 + +Total # of neighbors = 1043329 +Ave neighs/atom = 579.62722 +Ave special neighs/atom = 2.0000000 +Neighbor list builds = 4470 +Dangerous builds = 9 +PPPM initialization ... + using 12-bit tables for long-range coulomb (src/lammps/src/kspace.cpp:339) + G vector (1/distance) = 0.25102555 + grid = 18 18 18 + stencil order = 5 + estimated absolute RMS force accuracy = 0.0035374557 + estimated relative force accuracy = 1.0652943e-05 + using double precision FFTW3 + 3d grid and FFT values/proc = 3328 648 +FEP settings ... + temperature = 300.000000 + tail no + 1-1 charge + 2-2 charge +Setting up Verlet run ... + Unit style : real + Current step : 0 + Time step : 1 +Per MPI rank memory allocation (min/avg/max) = 8.592 | 8.613 | 8.629 Mbytes +Step TotEng KinEng PotEng E_vdwl E_coul E_long Temp Press Density v_lambda v_qO v_qH + 0 -5538.2733 1254.8684 -6793.1417 1343.4731 21794.974 -30342.64 300.91662 -475.71492 1.0050909 0 -0 0 + 5000 -5579.9374 1246.75 -6826.6874 1443.8973 21614.133 -30343.039 298.96983 785.16037 1.0322471 0.0025 -0.002119 0.0010595 + 10000 -5528.9995 1276.273 -6805.2725 1375.6345 21734.24 -30343.779 306.04943 -41.209531 1.0207823 0.005 -0.004238 0.002119 + 15000 -5506.4227 1270.5724 -6776.9951 1356.298 21760.893 -30339.74 304.68242 -483.58254 1.0055775 0.0075 -0.006357 0.0031785 + 20000 -5520.1985 1271.766 -6791.9645 1413.5454 21705.741 -30340.672 304.96864 -25.849632 0.9997261 0.01 -0.008476 0.004238 + 25000 -5538.0793 1260.547 -6798.6264 1362.2032 21772.012 -30341.973 302.27834 -32.008776 1.0238659 0.0125 -0.010595 0.0052975 + 30000 -5519.1594 1267.066 -6786.2254 1327.6996 21784.187 -30341.191 303.84159 -919.77668 0.99978921 0.015 -0.012714 0.006357 + 35000 -5565.8849 1244.6201 -6810.505 1384.6297 21700.099 -30337.317 298.45908 -41.645325 1.0154185 0.0175 -0.014833 0.0074165 + 40000 -5610.4882 1225.8647 -6836.3529 1416.9133 21661.893 -30343.456 293.96154 86.229199 1.0091158 0.02 -0.016952 0.008476 + 45000 -5609.3562 1208.1008 -6817.457 1387.4135 21711.193 -30343.791 289.70178 -101.30213 1.0155204 0.0225 -0.019071 0.0095355 + 50000 -5512.9435 1272.9818 -6785.9253 1379.4084 21746.829 -30342.184 305.2602 -12.658236 1.0143328 0.025 -0.02119 0.010595 + 55000 -5568.845 1249.4227 -6818.2677 1397.4685 21704.581 -30340.419 299.61074 -409.12372 0.99849506 0.0275 -0.023309 0.0116545 + 60000 -5519.3972 1216.5292 -6735.9264 1369.8115 21830.467 -30342.642 291.72291 117.70437 1.0138535 0.03 -0.025428 0.012714 + 65000 -5525.0539 1229.5034 -6754.5573 1354.6275 21755.174 -30340.101 294.8341 -462.0909 1.0102225 0.0325 -0.027547 0.0137735 + 70000 -5472.0836 1237.9936 -6710.0772 1352.7747 21872.795 -30340.246 296.87006 -5.9707079 1.010185 0.035 -0.029666 0.014833 + 75000 -5514.8966 1241.7822 -6756.6788 1369.2902 21788.901 -30344.123 297.77855 -36.505421 1.0055449 0.0375 -0.031785 0.0158925 + 80000 -5501.5809 1244.4839 -6746.0648 1406.1414 21762.858 -30338.116 298.42642 356.6391 1.0044866 0.04 -0.033904 0.016952 + 85000 -5522.8846 1243.4435 -6766.3281 1277.631 21834.13 -30341.071 298.17693 -1501.0557 1.0030405 0.0425 -0.036023 0.0180115 + 90000 -5521.2512 1226.9606 -6748.2118 1426.6301 21707.584 -30342.842 294.22434 431.61937 1.0037325 0.045 -0.038142 0.019071 + 95000 -5606.4261 1239.7565 -6846.1826 1458.2919 21603.577 -30341.265 297.2928 452.3491 1.0064454 0.0475 -0.040261 0.0201305 + 100000 -5526.6029 1244.4811 -6771.084 1442.7891 21665.95 -30338.28 298.42575 499.20413 1.0045851 0.05 -0.04238 0.02119 + 105000 -5589.4951 1234.4407 -6823.9358 1418.2214 21663.025 -30341.483 296.01807 101.88725 1.0139383 0.0525 -0.044499 0.0222495 + 110000 -5511.859 1262.1425 -6774.0015 1353.5279 21783.245 -30339.283 302.66093 -199.18931 1.0257755 0.055 -0.046618 0.023309 + 115000 -5647.3449 1229.413 -6876.758 1408.5179 21641.362 -30341.598 294.81243 -298.02921 1.0020969 0.0575 -0.048737 0.0243685 + 120000 -5449.7674 1281.3875 -6731.155 1304.5518 21888.494 -30341.641 307.27589 -694.089 1.0046108 0.06 -0.050856 0.025428 + 125000 -5430.3553 1247.5854 -6677.9407 1342.5284 21899.097 -30343.376 299.17016 125.17161 1.0091105 0.0625 -0.052975 0.0264875 + 130000 -5604.2229 1198.864 -6803.0869 1343.9437 21741.913 -30342.105 287.4868 -1029.1103 0.99252488 0.065 -0.055094 0.027547 + 135000 -5550.2149 1233.8869 -6784.1018 1298.4466 21844.496 -30341.697 295.88527 -1083.9932 1.0128891 0.0675 -0.057213 0.0286065 + 140000 -5526.9114 1246.4353 -6773.3466 1346.6874 21790.011 -30343.164 298.89435 -384.65885 1.0089922 0.07 -0.059332 0.029666 + 145000 -5594.7954 1213.2963 -6808.0917 1391.4251 21671.653 -30341.641 290.94766 -59.474002 1.0195886 0.0725 -0.061451 0.0307255 + 150000 -5580.5229 1259.8574 -6840.3803 1384.5548 21703.974 -30342.483 302.11297 -91.974674 1.0257584 0.075 -0.06357 0.031785 + 155000 -5566.2712 1230.138 -6796.4093 1366.5696 21707.576 -30336.313 294.98629 -482.01244 1.0070735 0.0775 -0.065689 0.0328445 + 160000 -5491.8647 1293.2326 -6785.0974 1386.5259 21718.735 -30343.487 310.11633 -89.376893 1.0093767 0.08 -0.067808 0.033904 + 165000 -5531.3681 1279.7729 -6811.1409 1354.4198 21753.203 -30342.651 306.88869 -392.95774 1.0134907 0.0825 -0.069927 0.0349635 + 170000 -5594.2308 1213.4715 -6807.7023 1482.6412 21600.249 -30341.895 290.98966 928.50126 1.0132201 0.085 -0.072046 0.036023 + 175000 -5519.3482 1257.2925 -6776.6407 1420.4909 21726.753 -30340.213 301.49792 254.94623 0.98901793 0.0875 -0.074165 0.0370825 + 180000 -5426.929 1250.4297 -6677.3587 1356.866 21886.349 -30343.3 299.85221 -245.8457 0.98302571 0.09 -0.076284 0.038142 + 185000 -5512.2746 1245.7072 -6757.9817 1380.9844 21794.2 -30342.884 298.71976 149.05128 1.0150091 0.0925 -0.078403 0.0392015 + 190000 -5449.1514 1266.5892 -6715.7406 1324.6562 21889.348 -30342.468 303.72725 -620.02591 1.0004189 0.095 -0.080522 0.040261 + 195000 -5479.3945 1315.5655 -6794.96 1485.0588 21649.278 -30344.957 315.47174 1314.4492 1.0231026 0.0975 -0.082641 0.0413205 + 200000 -5563.7047 1217.2005 -6780.9051 1358.0191 21745.737 -30342.731 291.88386 -588.88047 1.0064764 0.1 -0.08476 0.04238 + 205000 -5580.3808 1230.1745 -6810.5553 1411.4424 21686.461 -30344.166 294.99504 214.77057 1.0029345 0.1025 -0.086879 0.0434395 + 210000 -5498.3534 1242.7043 -6741.0576 1370.6141 21796.013 -30341.015 297.99966 -26.716679 1.0117516 0.105 -0.088998 0.044499 + 215000 -5458.5956 1250.9454 -6709.541 1395.6557 21783.461 -30338.225 299.97588 406.41699 1.0137518 0.1075 -0.091117 0.0455585 + 220000 -5626.602 1214.9754 -6841.5774 1473.3289 21571.914 -30341.748 291.3503 599.47976 1.0096167 0.11 -0.093236 0.046618 + 225000 -5600.5356 1224.9039 -6825.4395 1342.4891 21754.804 -30341.025 293.73115 -626.27228 1.0198862 0.1125 -0.095355 0.0476775 + 230000 -5454.9477 1275.1464 -6730.0941 1372.8993 21798.515 -30340.963 305.77926 154.34847 1.0029435 0.115 -0.097474 0.048737 + 235000 -5481.9555 1227.4931 -6709.4486 1352.587 21840.233 -30343.313 294.35203 -208.20676 1.0008406 0.1175 -0.099593 0.0497965 + 240000 -5558.573 1218.8864 -6777.4595 1385.8452 21752.45 -30343.537 292.28816 162.10969 1.0215047 0.12 -0.101712 0.050856 + 245000 -5520.9207 1225.4691 -6746.3898 1338.4141 21841.905 -30342.557 293.86667 -587.07136 0.99909027 0.1225 -0.103831 0.0519155 + 250000 -5469.8293 1274.8021 -6744.6315 1384.4128 21775.904 -30342.629 305.69671 64.465989 1.0105027 0.125 -0.10595 0.052975 + 255000 -5583.4632 1209.565 -6793.0282 1426.1758 21667.168 -30342.466 290.0529 515.82984 1.0156235 0.1275 -0.108069 0.0540345 + 260000 -5536.1405 1237.073 -6773.2135 1357.8241 21771.52 -30340.823 296.64929 -438.57147 1.0071049 0.13 -0.110188 0.055094 + 265000 -5611.0013 1237.3831 -6848.3844 1417.6236 21598.469 -30339.452 296.72365 -367.70672 0.99925355 0.1325 -0.112307 0.0561535 + 270000 -5551.1345 1227.3483 -6778.4827 1430.3281 21692.104 -30340.818 294.31731 287.00159 1.0010721 0.135 -0.114426 0.057213 + 275000 -5602.2651 1241.1358 -6843.4009 1391.9031 21673.697 -30343.58 297.62355 -264.79926 1.014186 0.1375 -0.116545 0.0582725 + 280000 -5505.2292 1277.7478 -6782.9769 1380.5349 21747.333 -30343.992 306.40307 39.871379 1.0224709 0.14 -0.118664 0.059332 + 285000 -5527.7736 1230.4336 -6758.2072 1397.1892 21732.299 -30342.417 295.05716 276.95418 1.0126839 0.1425 -0.120783 0.0603915 + 290000 -5450.4061 1265.2269 -6715.6329 1360.3141 21842.086 -30344 303.40057 207.59349 1.0147077 0.145 -0.122902 0.061451 + 295000 -5617.5993 1217.4105 -6835.0098 1487.1573 21606.767 -30340.365 291.93424 808.20982 1.0058589 0.1475 -0.125021 0.0625105 + 300000 -5543.8664 1236.4411 -6780.3075 1377.4792 21755.298 -30344.078 296.49776 -182.45633 1.0097782 0.15 -0.12714 0.06357 + 305000 -5609.1426 1227.6292 -6836.7719 1437.6073 21603.923 -30343.337 294.38468 575.20852 1.0196556 0.1525 -0.129259 0.0646295 + 310000 -5549.1828 1228.5889 -6777.7717 1394.6395 21691.45 -30340.369 294.61482 348.39727 1.0352996 0.155 -0.131378 0.065689 + 315000 -5535.243 1248.2444 -6783.4874 1342.2366 21792.192 -30343.483 299.32819 -625.73677 1.0050826 0.1575 -0.133497 0.0667485 + 320000 -5534.3687 1255.4471 -6789.8158 1391.0816 21739.717 -30344.081 301.05539 302.16436 1.0216155 0.16 -0.135616 0.067808 + 325000 -5471.3922 1279.3669 -6750.759 1350.9981 21795.148 -30342.392 306.79133 -197.30703 1.0143436 0.1625 -0.137735 0.0688675 + 330000 -5598.8232 1226.4651 -6825.2883 1398.7718 21677.079 -30344.7 294.10553 -213.61646 1.0148836 0.165 -0.139854 0.069927 + 335000 -5551.2529 1259.6937 -6810.9465 1387.3662 21685.187 -30344.308 302.07371 138.36812 1.0229544 0.1675 -0.141973 0.0709865 + 340000 -5614.9057 1204.9401 -6819.8458 1400.0625 21668.036 -30343.31 288.94385 -386.78257 0.99685483 0.17 -0.144092 0.072046 + 345000 -5520.3252 1245.9691 -6766.2942 1428.7502 21696.415 -30345.482 298.78256 768.45616 1.0192108 0.1725 -0.146211 0.0731055 + 350000 -5513.6753 1249.7279 -6763.4032 1284.3535 21864.411 -30343.729 299.68392 -1040.9933 1.0093189 0.175 -0.14833 0.074165 + 355000 -5547.2549 1244.5392 -6791.7941 1428.3735 21662.08 -30340.69 298.43968 593.10891 1.0193921 0.1775 -0.150449 0.0752245 + 360000 -5589.7133 1245.8181 -6835.5314 1358.8909 21676.184 -30346.212 298.74637 -615.61801 1.0169406 0.18 -0.152568 0.076284 + 365000 -5487.1246 1223.3024 -6710.427 1333.6832 21894.405 -30339.2 293.34711 -385.63992 0.99614598 0.1825 -0.154687 0.0773435 + 370000 -5557.3434 1261.7301 -6819.0735 1430.6594 21657.49 -30341.935 302.56204 256.97472 1.0130616 0.185 -0.156806 0.078403 + 375000 -5453.0587 1250.9921 -6704.0508 1417.9533 21782.943 -30342.475 299.98708 759.88581 1.005535 0.1875 -0.158925 0.0794625 + 380000 -5556.3547 1222.266 -6778.6207 1358.5536 21812.006 -30343.775 293.09858 -190.9931 1.0101874 0.19 -0.161044 0.080522 + 385000 -5501.4025 1265.3304 -6766.7328 1424.8468 21722.251 -30342.613 303.42538 676.19284 1.0082991 0.1925 -0.163163 0.0815815 + 390000 -5553.0922 1226.1248 -6779.217 1386.2701 21755.237 -30343.024 294.02392 -55.846714 1.0141611 0.195 -0.165282 0.082641 + 395000 -5464.7179 1292.5065 -6757.2245 1391.9142 21755.809 -30342.569 309.94221 143.63086 1.001399 0.1975 -0.167401 0.0837005 + 400000 -5552.2274 1226.6092 -6778.8365 1437.5354 21706.283 -30344.844 294.14007 383.64397 0.99717814 0.2 -0.16952 0.08476 + 405000 -5465.9964 1302.7208 -6768.7172 1272.3678 21868.849 -30344.642 312.39159 -1291.0003 1.0128426 0.2025 -0.171639 0.0858195 + 410000 -5502.6819 1234.6487 -6737.3307 1351.0308 21789.612 -30344.433 296.06795 -265.76646 1.0210657 0.205 -0.173758 0.086879 + 415000 -5633.7475 1231.4568 -6865.2043 1369.0122 21700.285 -30343.991 295.30253 -446.38607 1.0197817 0.2075 -0.175877 0.0879385 + 420000 -5501.9642 1301.7275 -6803.6917 1383.0254 21739.249 -30343.668 312.1534 167.94828 1.0154563 0.21 -0.177996 0.088998 + 425000 -5532.9109 1273.6513 -6806.5622 1353.034 21735.703 -30341.962 305.42074 -386.7674 1.0176249 0.2125 -0.180115 0.0900575 + 430000 -5610.4928 1229.6066 -6840.0994 1384.0534 21685.85 -30340.003 294.85885 -392.26296 1.0166946 0.215 -0.182234 0.091117 + 435000 -5485.4138 1292.946 -6778.3598 1331.4915 21809.953 -30343.379 310.04759 -684.00607 0.99728831 0.2175 -0.184353 0.0921765 + 440000 -5423.494 1275.8582 -6699.3523 1355.0802 21885.761 -30344.38 305.94996 -61.348045 1.0030785 0.22 -0.186472 0.093236 + 445000 -5543.7469 1262.7456 -6806.4925 1292.6629 21801.211 -30345.175 302.80557 -942.29675 1.019957 0.2225 -0.188591 0.0942955 + 450000 -5508.6756 1254.0646 -6762.7401 1336.3483 21793.153 -30345.515 300.72385 -424.30387 1.0135307 0.225 -0.19071 0.095355 + 455000 -5536.23 1241.0621 -6777.2921 1356.0154 21765.992 -30341.02 297.60588 -415.44019 1.013923 0.2275 -0.192829 0.0964145 + 460000 -5519.3214 1227.2661 -6746.5875 1350.5669 21793.876 -30345.691 294.2976 -838.27228 0.98835279 0.23 -0.194948 0.097474 + 465000 -5569.5398 1206.1409 -6775.6808 1341.3479 21779.493 -30347.065 289.2318 -1009.2997 0.98862235 0.2325 -0.197067 0.0985335 + 470000 -5477.9533 1252.9751 -6730.9284 1332.4765 21850.802 -30347.395 300.46261 -434.01812 1.0083102 0.235 -0.199186 0.099593 + 475000 -5454.9313 1285.8036 -6740.7349 1455.9404 21718.761 -30341.504 308.33485 1197.5128 1.018547 0.2375 -0.201305 0.1006525 + 480000 -5500.6228 1264.0646 -6764.6874 1416.1687 21720.5 -30344.339 303.12185 658.63999 1.0148714 0.24 -0.203424 0.101712 + 485000 -5487.2301 1267.1551 -6754.3852 1393.3235 21727.438 -30344.266 303.86296 203.85856 1.0140199 0.2425 -0.205543 0.1027715 + 490000 -5542.8719 1257.8965 -6800.7685 1425.7521 21626.135 -30342.879 301.64276 372.74667 1.0129528 0.245 -0.207662 0.103831 + 495000 -5528.489 1239.5803 -6768.0694 1497.5941 21586.069 -30343.329 297.25054 1242.1701 1.0035294 0.2475 -0.209781 0.1048905 + 500000 -5537.0419 1260.1365 -6797.1784 1368.6229 21717.765 -30341.664 302.17989 -282.78916 1.0246698 0.25 -0.2119 0.10595 + 505000 -5476.9332 1287.4238 -6764.357 1473.5981 21665.477 -30346.513 308.72338 1132.7688 1.0122147 0.2525 -0.214019 0.1070095 + 510000 -5467.851 1267.0655 -6734.9165 1378.6348 21778.869 -30344.441 303.84146 69.208146 1.0140451 0.255 -0.216138 0.108069 + 515000 -5551.9713 1245.7115 -6797.6828 1403.6528 21721.547 -30342.376 298.72079 364.60527 1.0219092 0.2575 -0.218257 0.1091285 + 520000 -5611.3897 1244.7924 -6856.1821 1408.031 21638.811 -30344.364 298.5004 190.73446 1.0278442 0.26 -0.220376 0.110188 + 525000 -5572.5188 1247.1919 -6819.7107 1379.85 21691.92 -30343.696 299.0758 -203.52852 1.0154091 0.2625 -0.222495 0.1112475 + 530000 -5490.5915 1254.3538 -6744.9453 1387.0893 21799.214 -30343.112 300.79321 143.30552 1.0061614 0.265 -0.224614 0.112307 + 535000 -5625.8769 1218.1946 -6844.0715 1520.0402 21545.685 -30344.513 292.12226 1447.1785 1.0166808 0.2675 -0.226733 0.1133665 + 540000 -5469.8413 1287.4258 -6757.2671 1334.5626 21829.046 -30343.092 308.72386 -673.21682 1.0017181 0.27 -0.228852 0.114426 + 545000 -5512.6863 1236.8449 -6749.5313 1318.7318 21858.112 -30344.481 296.5946 -863.71117 0.99129185 0.2725 -0.230971 0.1154855 + 550000 -5639.1715 1217.2608 -6856.4322 1434.9176 21593.482 -30346.132 291.89833 253.12123 1.0171626 0.275 -0.23309 0.116545 + 555000 -5519.0565 1267.6791 -6786.7356 1405.1048 21738.951 -30344.793 303.98862 48.101225 0.99694182 0.2775 -0.235209 0.1176045 + 560000 -5652.7174 1173.1335 -6825.851 1439.7187 21641.222 -30344.805 281.31664 253.14906 1.0090928 0.28 -0.237328 0.118664 + 565000 -5669.4656 1195.4457 -6864.9113 1456.9433 21573.732 -30345.136 286.66708 192.38017 1.0032964 0.2825 -0.239447 0.1197235 + 570000 -5497.9803 1237.3935 -6735.3738 1357.9081 21789.301 -30345.378 296.72614 -251.982 1.0069437 0.285 -0.241566 0.120783 + 575000 -5499.6888 1271.8265 -6771.5154 1443.195 21717.024 -30345.292 304.98316 940.92536 1.012527 0.2875 -0.243685 0.1218425 + 580000 -5490.5569 1271.9789 -6762.5359 1373.9183 21766.793 -30344.227 305.01971 17.578848 1.0172064 0.29 -0.245804 0.122902 + 585000 -5530.2852 1261.3697 -6791.6549 1423.9039 21686.202 -30347.296 302.47562 -11.504613 0.99116915 0.2925 -0.247923 0.1239615 + 590000 -5488.4278 1288.5886 -6777.0164 1468.0208 21640.734 -30343.206 309.00269 759.61005 1.0084625 0.295 -0.250042 0.125021 + 595000 -5588.7416 1194.6582 -6783.3998 1372.9575 21791.426 -30342.728 286.47825 -143.90808 1.0132126 0.2975 -0.252161 0.1260805 + 600000 -5563.6701 1271.4938 -6835.1639 1449.6769 21642.784 -30343.735 304.90337 625.81539 1.011506 0.3 -0.25428 0.12714 + 605000 -5488.2233 1258.2868 -6746.5101 1407.7504 21741.629 -30343.159 301.73634 510.16631 1.0132228 0.3025 -0.256399 0.1281995 + 610000 -5549.338 1238.8653 -6788.2032 1429.9285 21684.29 -30344.283 297.07907 123.93393 0.98486733 0.305 -0.258518 0.129259 + 615000 -5490.009 1293.1302 -6783.1393 1394.1407 21736.673 -30344.121 310.09178 15.132265 1.0008809 0.3075 -0.260637 0.1303185 + 620000 -5557.3108 1229.2075 -6786.5184 1428.7497 21675.082 -30342.962 294.76316 221.88021 0.99717009 0.31 -0.262756 0.131378 + 625000 -5471.4619 1265.6593 -6737.1212 1435.6918 21726.949 -30346.08 303.50427 1082.3293 1.0296014 0.3125 -0.264875 0.1324375 + 630000 -5596.2646 1223.2613 -6819.5259 1310.3452 21748.215 -30349.483 293.33726 -956.30387 1.0193588 0.315 -0.266994 0.133497 + 635000 -5562.8009 1236.4073 -6799.2082 1435.0639 21648.724 -30348.81 296.48966 760.70171 1.0231151 0.3175 -0.269113 0.1345565 + 640000 -5546.5984 1245.5875 -6792.1859 1391.0829 21720.571 -30344.208 298.69107 84.132969 1.0139372 0.32 -0.271232 0.135616 + 645000 -5527.7237 1239.7635 -6767.4873 1469.2669 21670.472 -30347.702 297.29448 905.19045 1.004227 0.3225 -0.273351 0.1366755 + 650000 -5486.0101 1261.5972 -6747.6073 1428.0768 21719.918 -30346.814 302.53018 872.03447 1.0211175 0.325 -0.27547 0.137735 + 655000 -5512.6624 1274.7595 -6787.4218 1407.8255 21738.349 -30345.234 305.68648 315.34392 1.0159005 0.3275 -0.277589 0.1387945 + 660000 -5530.8232 1277.0139 -6807.8371 1411.9654 21724.952 -30344.558 306.22709 228.19739 1.0073576 0.33 -0.279708 0.139854 + 665000 -5584.6646 1210.7846 -6795.4492 1315.201 21842.284 -30345.244 290.34534 -1036.0503 0.99766647 0.3325 -0.281827 0.1409135 + 670000 -5613.7042 1216.7094 -6830.4135 1469.5492 21569.502 -30344.552 291.7661 569.55799 1.0191265 0.335 -0.283946 0.141973 + 675000 -5403.7426 1262.8723 -6666.6149 1452.6942 21769.922 -30345.597 302.83594 882.59425 0.99095227 0.3375 -0.286065 0.1430325 + 680000 -5621.8921 1220.9976 -6842.8897 1408.834 21654.021 -30346.563 292.79442 -296.23945 1.0015159 0.34 -0.288184 0.144092 + 685000 -5488.859 1304.9527 -6793.8117 1364.554 21763.427 -30348.59 312.92679 -211.30923 1.0149918 0.3425 -0.290303 0.1451515 + 690000 -5504.5878 1248.5637 -6753.1515 1384.0982 21716.14 -30343.281 299.40474 -82.795916 1.0093095 0.345 -0.292422 0.146211 + 695000 -5596.2796 1229.4773 -6825.7569 1344.4706 21742.04 -30345.718 294.82784 -1005.4356 1.0046317 0.3475 -0.294541 0.1472705 + 700000 -5532.1829 1295.7513 -6827.9341 1347.6545 21694.469 -30347.727 310.7203 -803.09684 1.0085103 0.35 -0.29666 0.14833 + 705000 -5556.5407 1230.5343 -6787.075 1344.8187 21812.434 -30348.843 295.08131 -201.00712 1.010319 0.3525 -0.298779 0.1493895 + 710000 -5633.8136 1234.4193 -6868.2329 1333.0942 21730.242 -30348.833 296.01293 -1368.0288 1.0000966 0.355 -0.300898 0.150449 + 715000 -5631.0856 1227.5912 -6858.6768 1395.4568 21681.852 -30346.637 294.37556 -255.61879 1.0144939 0.3575 -0.303017 0.1515085 + 720000 -5549.3688 1210.7798 -6760.1486 1418.7021 21758.926 -30350.326 290.3442 536.38975 1.0085621 0.36 -0.305136 0.152568 + 725000 -5628.8043 1231.3533 -6860.1576 1458.785 21573.579 -30350.207 295.27771 707.01861 1.0187202 0.3625 -0.307255 0.1536275 + 730000 -5548.9622 1251.7202 -6800.6824 1421.0395 21671.018 -30346.928 300.16167 167.64307 1.0191958 0.365 -0.309374 0.154687 + 735000 -5500.5941 1264.5064 -6765.1005 1364.4388 21811.902 -30349.578 303.22781 -108.03631 1.0186648 0.3675 -0.311493 0.1557465 + 740000 -5558.7077 1258.2914 -6816.9991 1409.4089 21709.041 -30348.945 301.73744 275.48329 1.0184094 0.37 -0.313612 0.156806 + 745000 -5482.8834 1276.9878 -6759.8711 1311.3195 21878.96 -30349.341 306.22082 -437.5413 1.0200645 0.3725 -0.315731 0.1578655 + 750000 -5535.8763 1236.3636 -6772.2398 1319.5529 21810.776 -30344.948 296.47917 -956.45661 1.0011678 0.375 -0.31785 0.158925 + 755000 -5533.7191 1213.8761 -6747.5953 1306.0985 21808.043 -30348.582 291.08669 -683.27383 1.0227604 0.3775 -0.319969 0.1599845 + 760000 -5429.3835 1303.7005 -6733.084 1333.5357 21913.252 -30348.286 312.62653 -151.87516 1.0091534 0.38 -0.322088 0.161044 + 765000 -5667.794 1212.9659 -6880.7599 1506.7874 21503.139 -30348.871 290.86842 891.10492 1.0215444 0.3825 -0.324207 0.1621035 + 770000 -5484.9356 1311.0182 -6795.9538 1435.0311 21677.068 -30348.949 314.3813 620.96832 1.0136132 0.385 -0.326326 0.163163 + 775000 -5518.6285 1232.7734 -6751.4018 1411.338 21740.98 -30346.257 295.61824 753.30258 1.0333129 0.3875 -0.328445 0.1642225 + 780000 -5537.1259 1217.4396 -6754.5655 1316.3755 21845.361 -30345.727 291.94122 -662.73038 1.0188948 0.39 -0.330564 0.165282 + 785000 -5604.5939 1277.8467 -6882.4407 1444.6878 21553.248 -30348.7 306.4268 227.68388 1.0209541 0.3925 -0.332683 0.1663415 + 790000 -5462.4478 1319.1458 -6781.5936 1327.6125 21815.004 -30349.006 316.33029 -750.24035 1.003079 0.395 -0.334802 0.167401 + 795000 -5487.7929 1318.8959 -6806.6888 1389.8939 21723.298 -30346.719 316.27037 -264.12282 0.99738096 0.3975 -0.336921 0.1684605 + 800000 -5523.8042 1255.7477 -6779.5519 1396.5496 21737.296 -30349.897 301.12747 178.89364 1.007163 0.4 -0.33904 0.16952 + 805000 -5538.132 1228.7841 -6766.9161 1453.7899 21669.265 -30350.684 294.66162 1049.0776 1.0297174 0.4025 -0.341159 0.1705795 + 810000 -5627.0669 1252.1172 -6879.1841 1502.1084 21549.483 -30350.675 300.25688 983.16188 1.0119201 0.405 -0.343278 0.171639 + 815000 -5573.0687 1226.8854 -6799.9541 1394.9184 21727.977 -30347.963 294.20631 -195.24823 0.99836901 0.4075 -0.345397 0.1726985 + 820000 -5621.5966 1294.197 -6915.7936 1463.2446 21542.129 -30349.926 310.3476 619.5463 1.0271072 0.41 -0.347516 0.173758 + 825000 -5616.7109 1249.8626 -6866.5734 1421.5256 21635.884 -30349.527 299.71622 -263.11411 0.99748321 0.4125 -0.349635 0.1748175 + 830000 -5471.708 1265.0891 -6736.7971 1353.2361 21810.332 -30347.83 303.36753 -108.71402 1.0163163 0.415 -0.351754 0.175877 + 835000 -5593.3653 1226.0643 -6819.4296 1488.2202 21624.762 -30347.972 294.0094 1267.159 1.0140919 0.4175 -0.353873 0.1769365 + 840000 -5436.1751 1245.9424 -6682.1175 1341.7719 21874.67 -30349.816 298.77616 37.078091 1.0076819 0.42 -0.355992 0.177996 + 845000 -5557.2162 1246.2846 -6803.5008 1365.7244 21760.327 -30351.424 298.85822 -504.55518 0.9986108 0.4225 -0.358111 0.1790555 + 850000 -5500.2978 1263.3648 -6763.6626 1345.3152 21797.277 -30346.719 302.95405 -493.77227 1.0071264 0.425 -0.36023 0.180115 + 855000 -5532.9745 1235.4096 -6768.3841 1356.9198 21790.77 -30352.767 296.25042 -221.24996 1.0158146 0.4275 -0.362349 0.1811745 + 860000 -5473.4178 1255.0438 -6728.4615 1383.6228 21792.167 -30348.034 300.95866 280.22925 1.0186752 0.43 -0.364468 0.182234 + 865000 -5603.437 1240.4488 -6843.8858 1465.1006 21616.501 -30349.222 297.45881 654.55077 1.007261 0.4325 -0.366587 0.1832935 + 870000 -5499.3395 1276.346 -6775.6855 1440.1417 21672.034 -30350.435 306.06692 628.16253 1.0139482 0.435 -0.368706 0.184353 + 875000 -5513.1545 1243.4424 -6756.5968 1375.4559 21786.838 -30347.817 298.17666 -219.76957 0.99017833 0.4375 -0.370825 0.1854125 + 880000 -5472.5421 1280.3435 -6752.8857 1389.4586 21788.568 -30351.891 307.02554 154.88101 1.0054199 0.44 -0.372944 0.186472 + 885000 -5450.9797 1258.2665 -6709.2462 1459.3097 21782.134 -30350.183 301.73147 980.80415 0.98845289 0.4425 -0.375063 0.1875315 + 890000 -5484.1944 1269.9087 -6754.1031 1387.4264 21765.016 -30349.116 304.52326 204.84716 1.0246385 0.445 -0.377182 0.188591 + 895000 -5581.7831 1248.4268 -6830.2099 1369.3871 21724.547 -30352.181 299.37193 -414.06005 1.0149078 0.4475 -0.379301 0.1896505 + 900000 -5458.4222 1232.2813 -6690.7035 1346.4094 21839.134 -30347.541 295.50024 -50.890974 1.0146972 0.45 -0.38142 0.19071 + 905000 -5513.276 1295.0801 -6808.3561 1376.4555 21731.514 -30351.543 310.55936 311.8453 1.0351056 0.4525 -0.383539 0.1917695 + 910000 -5573.436 1223.655 -6797.091 1400.1726 21683.422 -30350.684 293.43167 236.51663 1.0328044 0.455 -0.385658 0.192829 + 915000 -5533.4538 1223.4571 -6756.9109 1289.5678 21854.813 -30350.499 293.38421 -1263.7914 1.0023563 0.4575 -0.387777 0.1938885 + 920000 -5460.2089 1299.6804 -6759.8892 1426.0806 21734.807 -30345.969 311.6625 660.70504 1.0188703 0.46 -0.389896 0.194948 + 925000 -5586.3754 1190.3746 -6776.75 1370.4255 21793.33 -30353.014 285.45105 -349.07473 1.0001717 0.4625 -0.392015 0.1960075 + 930000 -5495.2747 1270.1927 -6765.4674 1366.3034 21771.462 -30352.631 304.59137 -253.14407 1.0062704 0.465 -0.394134 0.197067 + 935000 -5611.8812 1230.765 -6842.6462 1391.1109 21660.135 -30351.536 295.13663 -220.68043 1.0218857 0.4675 -0.396253 0.1981265 + 940000 -5573.1587 1243.9588 -6817.1176 1382.0507 21706.984 -30350.576 298.30051 -180.40488 1.0138507 0.47 -0.398372 0.199186 + 945000 -5443.1533 1293.3755 -6736.5288 1382.9147 21798.711 -30351.85 310.15058 147.36979 1.0066036 0.4725 -0.400491 0.2002455 + 950000 -5592.1333 1210.312 -6802.4453 1418.411 21688.66 -30351.52 290.23203 428.64993 1.0142914 0.475 -0.40261 0.201305 + 955000 -5470.1207 1296.1018 -6766.2225 1384.9375 21787.214 -30352.361 310.80436 433.7878 1.0216208 0.4775 -0.404729 0.2023645 + 960000 -5526.56 1277.3546 -6803.9145 1435.0129 21692.393 -30354.825 306.30878 456.96466 1.0098621 0.48 -0.406848 0.203424 + 965000 -5547.3226 1267.7833 -6815.1059 1336.807 21754.287 -30354.401 304.01361 -806.5152 1.0089205 0.4825 -0.408967 0.2044835 + 970000 -5590.3411 1253.2629 -6843.604 1436.0627 21619.623 -30354.576 300.53161 744.58683 1.0362403 0.485 -0.411086 0.205543 + 975000 -5495.8789 1288.8517 -6784.7306 1379.6531 21754.168 -30353.143 309.0658 56.414932 1.0194472 0.4875 -0.413205 0.2066025 + 980000 -5626.672 1235.0454 -6861.7174 1422.039 21639.365 -30353.345 296.16306 266.29939 1.0212144 0.49 -0.415324 0.207662 + 985000 -5517.2302 1266.6674 -6783.8976 1419.6659 21739.847 -30353.802 303.746 170.35635 1.0008605 0.4925 -0.417443 0.2087215 + 990000 -5615.6311 1235.67 -6851.301 1498.5618 21566.364 -30353.27 296.31284 929.92737 1.003868 0.495 -0.419562 0.209781 + 995000 -5579.2851 1215.2415 -6794.5265 1446.7531 21659.857 -30349.867 291.4141 803.74446 1.0190058 0.4975 -0.421681 0.2108405 + 1000000 -5581.9464 1263.6254 -6845.5718 1438.0232 21619.891 -30351.248 303.01654 385.93353 1.0173448 0.5 -0.4238 0.2119 + 1005000 -5587.1547 1249.0387 -6836.1934 1366.6385 21760.49 -30352.889 299.51867 -349.59036 1.0129845 0.5025 -0.425919 0.2129595 + 1010000 -5492.8452 1249.3309 -6742.1761 1343.9605 21845.845 -30357.274 299.58872 -278.05905 1.0062862 0.505 -0.428038 0.214019 + 1015000 -5502.1939 1259.7886 -6761.9825 1450.3663 21691.043 -30355.864 302.09648 884.22091 1.0091653 0.5075 -0.430157 0.2150785 + 1020000 -5519.7794 1263.3012 -6783.0806 1348.5309 21772.103 -30355.053 302.93879 -485.09205 1.0162589 0.51 -0.432276 0.216138 + 1025000 -5492.2145 1240.7923 -6733.0068 1347.9912 21829.98 -30355.146 297.54118 -129.21819 1.0122922 0.5125 -0.434395 0.2171975 + 1030000 -5481.098 1251.514 -6732.612 1349.3618 21805.918 -30351.771 300.11222 -294.39997 1.015786 0.515 -0.436514 0.218257 + 1035000 -5540.7907 1274.0244 -6814.8151 1422.1127 21668.122 -30353.396 305.51021 -9.7807193 0.99517879 0.5175 -0.438633 0.2193165 + 1040000 -5511.938 1311.5237 -6823.4617 1415.7285 21685.767 -30355.446 314.50251 557.98532 1.0269334 0.52 -0.440752 0.220376 + 1045000 -5482.845 1244.1463 -6726.9913 1334.7343 21892.686 -30356.188 298.34546 -188.18213 1.008059 0.5225 -0.442871 0.2214355 + 1050000 -5527.147 1257.8916 -6785.0386 1365.7652 21734.325 -30350.269 301.64158 -259.9832 1.0161046 0.525 -0.44499 0.222495 + 1055000 -5539.8717 1252.0619 -6791.9335 1333.1576 21792.877 -30357.118 300.24361 -478.89657 1.0214282 0.5275 -0.447109 0.2235545 + 1060000 -5495.1427 1262.1608 -6757.3034 1324.8343 21817.102 -30350.41 302.66532 -759.6781 1.0013723 0.53 -0.449228 0.224614 + 1065000 -5521.9323 1269.1474 -6791.0797 1383.5193 21729.979 -30354.504 304.3407 -62.142854 1.0143611 0.5325 -0.451347 0.2256735 + 1070000 -5561.1075 1234.4671 -6795.5746 1444.645 21667.151 -30353.573 296.0244 539.71717 1.0090477 0.535 -0.453466 0.226733 + 1075000 -5456.2884 1263.0452 -6719.3336 1372.1813 21849.919 -30353.173 302.8774 136.37572 1.0050362 0.5375 -0.455585 0.2277925 + 1080000 -5454.7662 1277.8421 -6732.6083 1390.7236 21772.189 -30355.181 306.42569 290.51564 1.0135088 0.54 -0.457704 0.228852 + 1085000 -5520.75 1250.6045 -6771.3545 1425.3072 21768.597 -30353.777 299.89414 798.65189 1.0159181 0.5425 -0.459823 0.2299115 + 1090000 -5572.3388 1291.5598 -6863.8986 1367.7168 21667.215 -30354.683 309.71518 -450.87566 1.0262859 0.545 -0.461942 0.230971 + 1095000 -5529.6535 1255.2347 -6784.8882 1403.3219 21741.696 -30353.889 301.00445 -92.495589 0.99150484 0.5475 -0.464061 0.2320305 + 1100000 -5579.3489 1256.7239 -6836.0728 1453.5027 21616.876 -30355.538 301.36157 720.78396 1.0267204 0.55 -0.46618 0.23309 + 1105000 -5567.3822 1242.176 -6809.5582 1421.8053 21697.289 -30354.084 297.87299 289.72195 1.0148833 0.5525 -0.468299 0.2341495 + 1110000 -5547.6002 1257.4285 -6805.0287 1346.2645 21768.848 -30359.105 301.53052 -635.42562 1.0119993 0.555 -0.470418 0.235209 + 1115000 -5605.4545 1236.2432 -6841.6976 1429.3813 21606.101 -30358.883 296.45029 40.119249 1.0046973 0.5575 -0.472537 0.2362685 + 1120000 -5611.5649 1169.0166 -6780.5815 1332.8494 21815.405 -30357.613 280.3294 -562.91711 1.0167305 0.56 -0.474656 0.237328 + 1125000 -5640.062 1211.8843 -6851.9464 1418.7199 21673.242 -30357.067 290.60906 -62.63663 0.999031 0.5625 -0.476775 0.2383875 + 1130000 -5513.3831 1261.9318 -6775.315 1417.1438 21719.904 -30352.493 302.61042 310.56176 1.0135707 0.565 -0.478894 0.239447 + 1135000 -5549.8659 1301.5049 -6851.3708 1396.8697 21686.613 -30356.008 312.10001 31.758375 1.0173007 0.5675 -0.481013 0.2405065 + 1140000 -5523.4602 1224.9938 -6748.454 1343.5247 21842.717 -30354.173 293.75272 -528.56467 1.0040317 0.57 -0.483132 0.241566 + 1145000 -5557.8153 1233.2598 -6791.0752 1354.164 21797.958 -30358.703 295.73489 -493.27153 1.0032311 0.5725 -0.485251 0.2426255 + 1150000 -5534.5822 1243.3356 -6777.9177 1394.6839 21739.915 -30355.185 298.15105 152.31618 1.0071009 0.575 -0.48737 0.243685 + 1155000 -5534.8189 1242.3492 -6777.1682 1373.8187 21793.555 -30357.857 297.91453 37.731508 1.0083807 0.5775 -0.489489 0.2447445 + 1160000 -5479.2448 1282.7579 -6762.0027 1384.5519 21809.073 -30358.289 307.6045 -216.01178 0.99242248 0.58 -0.491608 0.245804 + 1165000 -5523.2438 1251.4259 -6774.6697 1415.3042 21701.997 -30352.822 300.0911 432.84941 1.0114966 0.5825 -0.493727 0.2468635 + 1170000 -5481.1255 1250.2894 -6731.4149 1365.5166 21830.221 -30359.589 299.81856 91.910339 1.0132882 0.585 -0.495846 0.247923 + 1175000 -5495.7532 1253.5429 -6749.2961 1424.9266 21793.408 -30356.209 300.59876 523.05665 1.0001084 0.5875 -0.497965 0.2489825 + 1180000 -5559.8569 1275.3404 -6835.1973 1409.3754 21651.571 -30355.789 305.82578 227.26668 1.022457 0.59 -0.500084 0.250042 + 1185000 -5542.5567 1221.3997 -6763.9563 1364.9044 21800.061 -30356.771 292.89083 -219.95244 1.0177023 0.5925 -0.502203 0.2511015 + 1190000 -5539.2712 1222.6707 -6761.9418 1388.3455 21753.974 -30354.881 293.19562 98.025711 1.0101339 0.595 -0.504322 0.252161 + 1195000 -5592.3472 1219.7396 -6812.0868 1415.1952 21710.597 -30354.347 292.49275 216.23076 1.0044744 0.5975 -0.506441 0.2532205 + 1200000 -5578.5334 1216.6352 -6795.1686 1303.7768 21836.822 -30356.422 291.74831 -732.00281 1.0186318 0.6 -0.50856 0.25428 + 1205000 -5487.1214 1272.9249 -6760.0463 1430.2949 21760.198 -30361.194 305.24654 976.44234 1.0227788 0.6025 -0.510679 0.2553395 + 1210000 -5535.2065 1305.0213 -6840.2278 1427.2627 21661.686 -30358.861 312.94326 552.57394 1.0203432 0.605 -0.512798 0.256399 + 1215000 -5603.1868 1222.0704 -6825.2571 1424.8423 21663.067 -30360.771 293.05167 43.886877 1.0009903 0.6075 -0.514917 0.2574585 + 1220000 -5592.1952 1236.0907 -6828.286 1409.2149 21703.364 -30359.131 296.41374 -65.380785 0.99854282 0.61 -0.517036 0.258518 + 1225000 -5568.7119 1244.7205 -6813.4324 1399.4565 21706.513 -30362.046 298.48315 270.33966 1.0281978 0.6125 -0.519155 0.2595775 + 1230000 -5477.0467 1241.6279 -6718.6746 1308.0744 21899.276 -30355.205 297.74155 -810.43376 1.0001488 0.615 -0.521274 0.260637 + 1235000 -5514.9454 1219.9538 -6734.8992 1367.2379 21833.29 -30355.096 292.54411 -112.9674 1.0106137 0.6175 -0.523393 0.2616965 + 1240000 -5521.6055 1224.654 -6746.2595 1359.0484 21804.839 -30358.991 293.67122 -103.13203 1.0121451 0.62 -0.525512 0.262756 + 1245000 -5521.627 1203.8833 -6725.5103 1345.4748 21859.062 -30356.508 288.69043 -551.04569 0.99671631 0.6225 -0.527631 0.2638155 + 1250000 -5500.3647 1258.1202 -6758.4849 1419.8081 21749.943 -30357.318 301.69639 266.72902 0.98901247 0.625 -0.52975 0.264875 + 1255000 -5552.125 1243.7071 -6795.8321 1455.2991 21666.462 -30358.89 298.24014 599.97186 1.0064842 0.6275 -0.531869 0.2659345 + 1260000 -5578.793 1204.3943 -6783.1873 1379.1701 21781.6 -30362.903 288.81297 -21.534061 1.0251033 0.63 -0.533988 0.266994 + 1265000 -5465.6374 1276.5741 -6742.2115 1336.784 21844.446 -30358.502 306.12162 -410.74102 1.0115187 0.6325 -0.536107 0.2680535 + 1270000 -5520.3861 1231.6592 -6752.0453 1394.2067 21774.812 -30362.74 295.35107 352.62523 1.014927 0.635 -0.538226 0.269113 + 1275000 -5616.2092 1265.908 -6882.1172 1384.7822 21664.341 -30360.472 303.56391 -682.5968 1.0081338 0.6375 -0.540345 0.2701725 + 1280000 -5532.4186 1237.2775 -6769.6961 1397.704 21760.205 -30359.579 296.69834 209.80411 1.0168583 0.64 -0.542464 0.271232 + 1285000 -5496.3981 1253.4279 -6749.826 1391.0922 21757.065 -30360.207 300.57117 342.16946 1.0226223 0.6425 -0.544583 0.2722915 + 1290000 -5611.5982 1256.0653 -6867.6635 1395.2492 21671.891 -30359.421 301.20364 -118.2094 1.0185165 0.645 -0.546702 0.273351 + 1295000 -5568.4527 1218.3517 -6786.8043 1384.42 21730.535 -30358.386 292.15993 -5.9864557 1.0110819 0.6475 -0.548821 0.2744105 + 1300000 -5549.8143 1285.6459 -6835.4602 1387.4514 21693.531 -30362.59 308.29704 -161.52516 1.0112493 0.65 -0.55094 0.27547 + 1305000 -5516.7616 1271.5601 -6788.3217 1334.904 21812.758 -30361.82 304.91926 -887.65758 0.99294523 0.6525 -0.553059 0.2765295 + 1310000 -5541.1764 1247.0758 -6788.2523 1358.4634 21773.518 -30361.438 299.04796 -277.80074 1.015369 0.655 -0.555178 0.277589 + 1315000 -5559.5242 1227.118 -6786.6423 1424.8735 21749.242 -30361.297 294.26209 365.81548 1.0090832 0.6575 -0.557297 0.2786485 + 1320000 -5500.4878 1275.2925 -6775.7803 1378.9646 21774.694 -30364.402 305.81431 90.139393 1.0119212 0.66 -0.559416 0.279708 + 1325000 -5621.0444 1209.1452 -6830.1896 1404.2928 21691.047 -30364.044 289.95221 -116.04384 1.0138418 0.6625 -0.561535 0.2807675 + 1330000 -5585.3052 1256.8226 -6842.1277 1303.2839 21785.369 -30364.807 301.38523 -1350.6305 1.0072541 0.665 -0.563654 0.281827 + 1335000 -5595.6814 1228.0426 -6823.724 1326.3132 21774.063 -30364.35 294.48382 -853.10172 1.0189157 0.6675 -0.565773 0.2828865 + 1340000 -5535.3184 1240.3856 -6775.704 1444.9762 21724.661 -30365.799 297.44365 769.15382 1.0062411 0.67 -0.567892 0.283946 + 1345000 -5561.2416 1262.5784 -6823.82 1384.9167 21733.829 -30364.316 302.76547 342.6146 1.0354454 0.6725 -0.570011 0.2850055 + 1350000 -5542.1497 1254.5753 -6796.725 1434.2699 21729.395 -30364.302 300.84634 419.79647 0.99912169 0.675 -0.57213 0.286065 + 1355000 -5595.989 1217.3701 -6813.3591 1387.8286 21699.818 -30361.622 291.92454 -89.640329 1.016903 0.6775 -0.574249 0.2871245 + 1360000 -5489.52 1276.1569 -6765.6769 1369.4965 21808.278 -30360.429 306.02157 161.97718 1.0232691 0.68 -0.576368 0.288184 + 1365000 -5469.7144 1225.7769 -6695.4914 1326.3513 21902.018 -30363.808 293.9405 -693.83304 0.99124467 0.6825 -0.578487 0.2892435 + 1370000 -5521.5216 1239.8048 -6761.3264 1430.0627 21735.529 -30363.095 297.30438 329.41994 0.98996333 0.685 -0.580606 0.290303 + 1375000 -5596.948 1213.5383 -6810.4863 1483.4218 21571.534 -30357.305 291.00568 558.04104 0.99161295 0.6875 -0.582725 0.2913625 + 1380000 -5514.9435 1280.2938 -6795.2372 1340.7447 21792.544 -30361.88 307.0136 -293.2026 1.0241366 0.69 -0.584844 0.292422 + 1385000 -5573.106 1248.5767 -6821.6826 1393.7875 21722.652 -30364.587 299.40787 -241.85628 1.0070126 0.6925 -0.586963 0.2934815 + 1390000 -5583.7305 1220.9377 -6804.6682 1386.3088 21744.845 -30364.846 292.78005 -167.94814 0.99821704 0.695 -0.589082 0.294541 + 1395000 -5469.5824 1236.0582 -6705.6406 1361.2754 21845.151 -30361.887 296.40594 -104.21347 0.99776431 0.6975 -0.591201 0.2956005 + 1400000 -5500.7115 1233.2318 -6733.9433 1333.948 21873.313 -30362.934 295.72817 -125.0822 1.0263809 0.7 -0.59332 0.29666 + 1405000 -5419.1812 1321.0256 -6740.2068 1354.0297 21833.304 -30367.647 316.78106 -233.8297 1.009048 0.7025 -0.595439 0.2977195 + 1410000 -5527.8617 1249.276 -6777.1377 1393.035 21770.812 -30366.621 299.57557 167.07416 1.0156062 0.705 -0.597558 0.298779 + 1415000 -5481.1169 1252.3487 -6733.4656 1344.2947 21829.773 -30368.721 300.31238 -461.85938 1.0036389 0.7075 -0.599677 0.2998385 + 1420000 -5532.8156 1260.4335 -6793.2491 1421.7199 21704.917 -30367.338 302.25113 343.60928 1.0162706 0.71 -0.601796 0.300898 + 1425000 -5547.9732 1268.0648 -6816.038 1421.7341 21700.228 -30367.368 304.08111 271.4221 1.0128974 0.7125 -0.603915 0.3019575 + 1430000 -5536.9027 1243.6801 -6780.5828 1325.1833 21856.026 -30368.533 298.23368 -388.7135 1.0267827 0.715 -0.606034 0.303017 + 1435000 -5570.6543 1244.2756 -6814.9299 1374.1183 21711.381 -30367.694 298.37647 25.769415 1.0319664 0.7175 -0.608153 0.3040765 + 1440000 -5503.2808 1262.2145 -6765.4953 1424.7984 21726.029 -30366.421 302.67821 576.10782 1.0091121 0.72 -0.610272 0.305136 + 1445000 -5462.886 1270.7887 -6733.6747 1318.6859 21898.228 -30366.918 304.7343 -387.50044 1.0149824 0.7225 -0.612391 0.3061955 + 1450000 -5470.7571 1281.4135 -6752.1705 1425.3723 21732.871 -30366.098 307.2821 620.59705 1.0146359 0.725 -0.61451 0.307255 + 1455000 -5527.4601 1291.9524 -6819.4125 1360.2258 21771.527 -30363.755 309.80933 -502.25523 0.99718998 0.7275 -0.616629 0.3083145 + 1460000 -5514.1742 1247.8209 -6761.9951 1358.6541 21820.8 -30366.742 299.22664 -380.01478 1.0041344 0.73 -0.618748 0.309374 + 1465000 -5514.7862 1253.7836 -6768.5698 1365.8682 21811.261 -30362.79 300.65649 -368.26563 1.0008437 0.7325 -0.620867 0.3104335 + 1470000 -5551.171 1264.5229 -6815.694 1397.4282 21740.047 -30368.185 303.23177 288.1239 1.0256045 0.735 -0.622986 0.311493 + 1475000 -5529.159 1251.168 -6780.327 1398.811 21758.549 -30367.782 300.02927 160.29449 1.0114735 0.7375 -0.625105 0.3125525 + 1480000 -5487.0134 1283.588 -6770.6014 1365.85 21759.173 -30366.624 307.80355 -416.12696 1.0020989 0.74 -0.627224 0.313612 + 1485000 -5577.8776 1228.4499 -6806.3275 1450.1943 21661.771 -30368.442 294.58148 789.14519 1.0270287 0.7425 -0.629343 0.3146715 + 1490000 -5579.8432 1242.7181 -6822.5612 1373.2235 21710.845 -30368.691 298.00297 -429.78923 1.0167997 0.745 -0.631462 0.315731 + 1495000 -5604.0722 1240.0278 -6844.1 1398.7081 21667.622 -30366.025 297.35784 -50.192922 1.0151605 0.7475 -0.633581 0.3167905 + 1500000 -5570.8776 1262.8638 -6833.7414 1422.788 21689.484 -30369.581 302.8339 322.73206 1.0163727 0.75 -0.6357 0.31785 + 1505000 -5470.0341 1254.1779 -6724.212 1340.4518 21833.054 -30370 300.75103 -533.26569 1.0015798 0.7525 -0.637819 0.3189095 + 1510000 -5533.0307 1269.7134 -6802.7441 1420.8678 21723.006 -30366.979 304.47642 744.80635 1.031796 0.755 -0.639938 0.319969 + 1515000 -5525.4081 1275.2107 -6800.6188 1359.3204 21747.642 -30370.363 305.7947 -613.78881 1.0095566 0.7575 -0.642057 0.3210285 + 1520000 -5545.6601 1227.8235 -6773.4836 1408.0411 21740.939 -30370.107 294.43126 125.8317 1.0064814 0.76 -0.644176 0.322088 + 1525000 -5594.124 1256.3887 -6850.5127 1394.7089 21672.392 -30369.461 301.28118 -327.86845 1.0067392 0.7625 -0.646295 0.3231475 + 1530000 -5658.6676 1272.6607 -6931.3282 1495.4729 21500.185 -30372.43 305.18319 248.65409 0.98959859 0.765 -0.648414 0.324207 + 1535000 -5577.6076 1216.0351 -6793.6428 1376.0471 21752.612 -30371.553 291.60442 -330.81608 0.9975352 0.7675 -0.650533 0.3252665 + 1540000 -5550.0025 1218.5844 -6768.5869 1384.3771 21773.165 -30370.531 292.21573 42.712468 1.0056433 0.77 -0.652652 0.326326 + 1545000 -5600.0577 1225.9997 -6826.0574 1428.3352 21661.528 -30369.738 293.99391 813.57834 1.0479661 0.7725 -0.654771 0.3273855 + 1550000 -5468.4914 1242.477 -6710.9684 1334.3582 21918.607 -30369.067 297.94517 -396.37556 1.0001104 0.775 -0.65689 0.328445 + 1555000 -5513.1271 1227.7497 -6740.8768 1382.337 21787.791 -30366.523 294.41357 83.122435 1.0076956 0.7775 -0.659009 0.3295045 + 1560000 -5612.1024 1202.1818 -6814.2841 1387.0642 21764.028 -30367.929 288.28239 -258.78553 1.0064708 0.78 -0.661128 0.330564 + 1565000 -5590.9616 1206.59 -6797.5517 1411.8792 21677.549 -30369.916 289.33949 239.57892 1.011968 0.7825 -0.663247 0.3316235 + 1570000 -5510.954 1275.3855 -6786.3395 1375.3033 21763.279 -30370.61 305.8366 -189.55593 1.0102055 0.785 -0.665366 0.332683 + 1575000 -5554.726 1215.8818 -6770.6078 1456.4645 21711.707 -30369.754 291.56766 837.90544 1.0133372 0.7875 -0.667485 0.3337425 + 1580000 -5484.0735 1274.8343 -6758.9078 1407.2972 21780.536 -30370.2 305.70443 -7.4418979 0.98152191 0.79 -0.669604 0.334802 + 1585000 -5591.9223 1219.6327 -6811.555 1360.9446 21785.502 -30369.385 292.46711 -301.66012 1.0180482 0.7925 -0.671723 0.3358615 + 1590000 -5514.2903 1237.4873 -6751.7776 1345.8443 21817.054 -30372.767 296.74864 -500.35329 1.0070005 0.795 -0.673842 0.336921 + 1595000 -5610.6145 1230.3441 -6840.9586 1511.4929 21579.699 -30370.822 295.0357 1400.7717 1.0243843 0.7975 -0.675961 0.3379805 + 1600000 -5491.408 1268.4415 -6759.8495 1321.4258 21860.885 -30367.203 304.17145 -684.17812 1.0109371 0.8 -0.67808 0.33904 + 1605000 -5605.9376 1229.4973 -6835.4349 1358.4846 21752.878 -30376.308 294.83264 -415.45398 1.0211259 0.8025 -0.680199 0.3400995 + 1610000 -5460.3646 1272.2569 -6732.6215 1287.6367 21919.033 -30377.556 305.08636 -1142.3466 1.0015738 0.805 -0.682318 0.341159 + 1615000 -5526.5568 1289.8001 -6816.3569 1429.3397 21650.996 -30374.701 309.29322 -24.570161 0.99788407 0.8075 -0.684437 0.3422185 + 1620000 -5535.2137 1227.5421 -6762.7558 1398.9652 21806.783 -30374.667 294.36379 27.416625 0.99409746 0.81 -0.686556 0.343278 + 1625000 -5561.6432 1218.8583 -6780.5015 1405.1751 21715.111 -30372.575 292.28141 18.274766 1.0106948 0.8125 -0.688675 0.3443375 + 1630000 -5489.0281 1289.6971 -6778.7251 1344.2314 21847.424 -30375.392 309.2685 -428.65338 1.0104018 0.815 -0.690794 0.345397 + 1635000 -5583.1919 1258.7285 -6841.9204 1364.5536 21720.023 -30373.909 301.84226 -558.27346 1.0230546 0.8175 -0.692913 0.3464565 + 1640000 -5569.4234 1276.2864 -6845.7098 1511.2486 21602.351 -30376.345 306.05264 598.06785 0.98381579 0.82 -0.695032 0.347516 + 1645000 -5542.9848 1213.5702 -6756.5551 1373.3517 21798.358 -30369.633 291.01334 -254.12194 1.0019034 0.8225 -0.697151 0.3485755 + 1650000 -5482.0112 1245.3133 -6727.3245 1334.5937 21862.137 -30375.083 298.6253 -577.0133 1.0037952 0.825 -0.69927 0.349635 + 1655000 -5512.6207 1273.3548 -6785.9755 1416.6802 21724.769 -30370.223 305.34964 601.75776 1.0281521 0.8275 -0.701389 0.3506945 + 1660000 -5550.057 1251.0976 -6801.1546 1395.4686 21699.418 -30372.345 300.01237 -129.12786 1.0148586 0.83 -0.703508 0.351754 + 1665000 -5460.2178 1235.1935 -6695.4113 1384.7074 21833.611 -30369.508 296.19859 197.73924 0.9919328 0.8325 -0.705627 0.3528135 + 1670000 -5577.3307 1185.7465 -6763.0772 1310.7085 21842.338 -30371.235 284.34123 -1255.4953 0.98878939 0.835 -0.707746 0.353873 + 1675000 -5561.8847 1224.9469 -6786.8316 1430.0792 21692.395 -30375.915 293.74146 119.11524 0.99573454 0.8375 -0.709865 0.3549325 + 1680000 -5653.7552 1260.9531 -6914.7083 1519.8378 21487.885 -30376.356 302.37573 809.64185 1.015909 0.84 -0.711984 0.355992 + 1685000 -5512.3473 1274.3359 -6786.6832 1417.2785 21701.118 -30376.61 305.58491 445.91621 1.0312005 0.8425 -0.714103 0.3570515 + 1690000 -5508.29 1269.2138 -6777.5038 1366.728 21782.662 -30375.482 304.35663 14.172211 1.0249747 0.845 -0.716222 0.358111 + 1695000 -5591.9917 1161.3619 -6753.3536 1403.8396 21761.545 -30371.687 278.49382 339.64543 1.0133803 0.8475 -0.718341 0.3591705 + 1700000 -5466.9507 1266.1673 -6733.118 1372.2532 21825.493 -30372.223 303.62608 82.443385 1.0170363 0.85 -0.72046 0.36023 + 1705000 -5565.484 1255.6155 -6821.0995 1404.2864 21697.72 -30377.201 301.09577 115.4614 1.0224858 0.8525 -0.722579 0.3612895 + 1710000 -5468.4747 1242.5333 -6711.008 1261.827 21976.931 -30377.884 297.95866 -1282.3719 1.0025311 0.855 -0.724698 0.362349 + 1715000 -5506.6405 1286.9184 -6793.5589 1333.9602 21778.671 -30374.791 308.60219 -870.86182 0.99767742 0.8575 -0.726817 0.3634085 + 1720000 -5565.6601 1243.2447 -6808.9048 1453.7071 21688.169 -30376.668 298.12926 837.91968 1.0193803 0.86 -0.728936 0.364468 + 1725000 -5466.8293 1256.0749 -6722.9042 1322.9564 21895.211 -30378.325 301.20593 -717.52415 0.99658437 0.8625 -0.731055 0.3655275 + 1730000 -5558.3242 1213.4742 -6771.7984 1337.7732 21800.919 -30378.895 290.99031 -210.06727 1.0310242 0.865 -0.733174 0.366587 + 1735000 -5502.8243 1261.8071 -6764.6314 1382.0545 21775.488 -30375.951 302.58051 -0.31427925 1.0034826 0.8675 -0.735293 0.3676465 + 1740000 -5406.8705 1274.6358 -6681.5063 1334.7641 21893.52 -30376.72 305.65682 -231.76275 1.0050648 0.87 -0.737412 0.368706 + 1745000 -5469.6519 1259.7569 -6729.4088 1437.2403 21699.918 -30375.369 302.08887 487.94198 1.0000255 0.8725 -0.739531 0.3697655 + 1750000 -5510.7038 1273.4592 -6784.163 1346.9061 21834.988 -30377.095 305.37468 -381.44532 0.9990362 0.875 -0.74165 0.370825 + 1755000 -5498.5771 1291.9682 -6790.5453 1416.649 21698.465 -30377.922 309.81311 434.97184 1.020573 0.8775 -0.743769 0.3718845 + 1760000 -5518.5384 1277.985 -6796.5234 1361.2079 21765.884 -30377.646 306.45996 -37.746266 1.0246582 0.88 -0.745888 0.372944 + 1765000 -5507.052 1273.8365 -6780.8885 1421.9667 21726.518 -30377.959 305.46515 410.30385 1.0112394 0.8825 -0.748007 0.3740035 + 1770000 -5596.6543 1216.1185 -6812.7727 1416.069 21695.778 -30380.248 291.6244 1.5263758 1.0080966 0.885 -0.750126 0.375063 + 1775000 -5623.4083 1285.876 -6909.2843 1440.2387 21582.885 -30381.001 308.35222 -35.250138 1.0049289 0.8875 -0.752245 0.3761225 + 1780000 -5573.4547 1219.3235 -6792.7782 1442.9632 21676.665 -30377.304 292.39298 471.41075 1.0053462 0.89 -0.754364 0.377182 + 1785000 -5602.6194 1217.9746 -6820.594 1410.7105 21746.864 -30378.721 292.0695 188.14379 1.0190973 0.8925 -0.756483 0.3782415 + 1790000 -5545.0248 1232.4378 -6777.4626 1420.0285 21741.196 -30377.214 295.53778 756.19164 1.0274418 0.895 -0.758602 0.379301 + 1795000 -5518.8988 1253.2508 -6772.1496 1346.248 21846.81 -30376.809 300.52871 -133.93703 1.0242741 0.8975 -0.760721 0.3803605 + 1800000 -5440.5825 1267.2474 -6707.8299 1236.5895 21971.421 -30375.638 303.88509 -1636.0141 0.99063571 0.9 -0.76284 0.38142 + 1805000 -5447.8315 1288.0343 -6735.8658 1315.8015 21891.886 -30383.047 308.86977 -729.116 1.0031825 0.9025 -0.764959 0.3824795 + 1810000 -5612.711 1242.5543 -6855.2653 1383.4906 21692.082 -30384.343 297.96369 -319.45292 1.0215442 0.905 -0.767078 0.383539 + 1815000 -5557.273 1220.1604 -6777.4334 1421.1515 21742.938 -30384.256 292.59367 520.03095 1.0215411 0.9075 -0.769197 0.3845985 + 1820000 -5530.2316 1228.8432 -6759.0748 1332.6717 21862.178 -30382.059 294.67578 -655.51399 1.0042955 0.91 -0.771316 0.385658 + 1825000 -5602.3037 1229.7047 -6832.0084 1468.9012 21615.952 -30382.906 294.88237 421.71792 1.0045724 0.9125 -0.773435 0.3867175 + 1830000 -5547.6226 1200.4018 -6748.0244 1330.1166 21900.521 -30384.116 287.85555 -636.22865 1.0018051 0.915 -0.775554 0.387777 + 1835000 -5653.9568 1210.6899 -6864.6467 1386.2577 21734.456 -30383.944 290.32264 -127.89376 1.023499 0.9175 -0.777673 0.3888365 + 1840000 -5547.0916 1252.5655 -6799.6571 1419.6833 21720.231 -30379.345 300.36437 501.3533 1.0256074 0.92 -0.779792 0.389896 + 1845000 -5515.4901 1289.6285 -6805.1187 1364.8493 21767.014 -30383.713 309.25207 -436.33388 1.0012885 0.9225 -0.781911 0.3909555 + 1850000 -5486.8798 1292.9914 -6779.8712 1380.8937 21790.721 -30383.253 310.05848 22.6166 1.0096908 0.925 -0.78403 0.392015 + 1855000 -5533.7045 1263.0308 -6796.7353 1395.9678 21748.064 -30382.076 302.87396 -284.93741 0.9863725 0.9275 -0.786149 0.3930745 + 1860000 -5484.2708 1261.0621 -6745.3329 1439.0946 21765.649 -30381.058 302.40186 615.26529 0.9985027 0.93 -0.788268 0.394134 + 1865000 -5599.3124 1260.6731 -6859.9855 1381.0638 21741.092 -30385.213 302.30858 -582.58143 1.0024906 0.9325 -0.790387 0.3951935 + 1870000 -5529.7237 1260.0702 -6789.7939 1384.0683 21758.475 -30383.423 302.16399 53.835245 1.0113827 0.935 -0.792506 0.396253 + 1875000 -5455.8429 1269.0311 -6724.8739 1395.2702 21814.233 -30381.4 304.31281 191.35208 0.99331435 0.9375 -0.794625 0.3973125 + 1880000 -5522.3775 1257.2498 -6779.6273 1422.2951 21728.811 -30385.222 301.48766 210.15712 1.0034809 0.94 -0.796744 0.398372 + 1885000 -5581.928 1248.1059 -6830.0339 1353.7983 21777.194 -30384.398 299.29497 -685.33819 1.0064471 0.9425 -0.798863 0.3994315 + 1890000 -5504.863 1257.5766 -6762.4396 1382.3075 21802.301 -30382.745 301.56605 233.93301 1.0136359 0.945 -0.800982 0.400491 + 1895000 -5519.8568 1258.7721 -6778.629 1381.0637 21801.818 -30381.149 301.85273 157.33428 1.0106136 0.9475 -0.803101 0.4015505 + 1900000 -5415.1036 1245.1225 -6660.2261 1321.1611 21968.266 -30378.535 298.57955 -87.316091 1.0149157 0.95 -0.80522 0.40261 + 1905000 -5486.4216 1290.1083 -6776.5299 1344.556 21824.3 -30385.345 309.36712 -170.76911 1.0220988 0.9525 -0.807339 0.4036695 + 1910000 -5612.2769 1206.579 -6818.8559 1361.0857 21798.985 -30389.316 289.33686 -679.6889 1.0021422 0.955 -0.809458 0.404729 + 1915000 -5587.0108 1277.8116 -6864.8224 1390.9876 21720.193 -30387.546 306.41838 -88.421087 1.026429 0.9575 -0.811577 0.4057885 + 1920000 -5536.6115 1211.5465 -6748.1579 1346.0005 21869.713 -30386.249 290.52804 -614.6848 0.99432544 0.96 -0.813696 0.406848 + 1925000 -5475.7378 1270.4733 -6746.2111 1363.7746 21799.571 -30386.112 304.65867 263.14406 1.0338166 0.9625 -0.815815 0.4079075 + 1930000 -5509.1127 1248.2489 -6757.3616 1334.6571 21873.781 -30385.968 299.32926 -529.83539 1.0071671 0.965 -0.817934 0.408967 + 1935000 -5548.4157 1253.3639 -6801.7796 1361.0982 21797.539 -30389.563 300.55583 -390.63414 1.0119705 0.9675 -0.820053 0.4100265 + 1940000 -5585.3752 1257.0833 -6842.4584 1341.862 21777.004 -30390.791 301.44773 -588.69504 1.0234228 0.97 -0.822172 0.411086 + 1945000 -5553.1361 1239.7945 -6792.9307 1392.65 21775.612 -30388.808 297.30191 401.98554 1.0324973 0.9725 -0.824291 0.4121455 + 1950000 -5608.302 1257.6453 -6865.9473 1385.8155 21666.265 -30389.484 301.58252 -258.45732 1.0221941 0.975 -0.82641 0.413205 + 1955000 -5597.8019 1196.2504 -6794.0523 1368.6962 21798.614 -30384.841 286.86006 -187.46622 1.015951 0.9775 -0.828529 0.4142645 + 1960000 -5558.9543 1259.8231 -6818.7774 1402.0765 21687.027 -30386.699 302.10476 -130.60372 1.0087181 0.98 -0.830648 0.415324 + 1965000 -5522.1401 1225.1335 -6747.2736 1385.1221 21794.241 -30386.219 293.78621 59.455921 1.0156338 0.9825 -0.832767 0.4163835 + 1970000 -5460.8469 1294.812 -6755.6589 1403.377 21789.971 -30387.038 310.49507 67.767671 0.99695881 0.985 -0.834886 0.417443 + 1975000 -5562.7016 1276.5904 -6839.292 1381.4683 21781.499 -30385.8 306.12554 -104.19545 1.0255706 0.9875 -0.837005 0.4185025 + 1980000 -5519.6646 1253.4527 -6773.1173 1464.4023 21698.419 -30387.375 300.57712 1035.4507 1.0129337 0.99 -0.839124 0.419562 + 1985000 -5465.34 1288.7347 -6754.0747 1312.8409 21917.192 -30385.618 309.03772 -632.51392 1.0142034 0.9925 -0.841243 0.4206215 + 1990000 -5579.8472 1232.0079 -6811.8551 1325.3706 21848.546 -30387.507 295.43468 -736.79929 1.0087938 0.995 -0.843362 0.421681 + 1995000 -5474.4313 1277.8474 -6752.2787 1377.0964 21856.577 -30388.53 306.42697 -223.13166 0.99592342 0.9975 -0.845481 0.4227405 + 2000000 -5530.4657 1243.1127 -6773.5784 1448.1254 21758.477 -30388.792 298.09761 750.74861 1.0102049 1 -0.8476 0.4238 +Loop time of 13807.9 on 12 procs for 2000000 steps with 1800 atoms + +Performance: 12.515 ns/day, 1.918 hours/ns, 144.845 timesteps/s +95.3% CPU use with 12 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 7308.6 | 8043.1 | 8855.3 | 457.7 | 58.25 +Bond | 24.229 | 25.717 | 27.246 | 21.9 | 0.19 +Kspace | 2227.1 | 3047.3 | 3772.6 | 744.0 | 22.07 +Neigh | 525.14 | 526.08 | 527.05 | 2.6 | 3.81 +Comm | 809.43 | 867.32 | 940.11 | 181.0 | 6.28 +Output | 0.033333 | 0.034507 | 0.043203 | 1.4 | 0.00 +Modify | 880.49 | 1047.6 | 1183.6 | 402.0 | 7.59 +Other | | 250.7 | | | 1.82 + +Nlocal: 150.000 ave 167 max 140 min +Histogram: 2 3 2 0 2 0 0 1 1 1 +Nghost: 6112.58 ave 6193 max 6037 min +Histogram: 3 0 2 0 0 3 1 1 0 2 +Neighs: 87266.4 ave 94192 max 79682 min +Histogram: 2 0 1 0 1 4 2 0 0 2 + +Total # of neighbors = 1047197 +Ave neighs/atom = 581.77611 +Ave special neighs/atom = 2.0000000 +Neighbor list builds = 89139 +Dangerous builds = 0 +Total wall time: 3:59:31 diff --git a/examples/USER/fep/SPCEhyd/fep01/in-fep01-lj.lmp b/examples/USER/fep/SPCEhyd/fep01/in-fep01-lj.lmp new file mode 100644 index 0000000000..2f9bdbb7c5 --- /dev/null +++ b/examples/USER/fep/SPCEhyd/fep01/in-fep01-lj.lmp @@ -0,0 +1,72 @@ +# created by fftool + +units real +boundary p p p + +atom_style full +bond_style harmonic +angle_style harmonic + +special_bonds lj/coul 0.0 0.0 0.5 + +# remove hybrid if not necessary +pair_style hybrid lj/cut/coul/long 12.0 12.0 lj/cut/soft 2 0.5 12.0 +pair_modify tail no +kspace_style pppm 1.0e-5 + +read_data data.lmp + +pair_coeff 1 1 lj/cut/soft 0.000000 1.000000 0.0 # Hwh Hwh +pair_coeff 1 2 lj/cut/soft 0.000000 1.000000 0.0 # Hwh Owh +pair_coeff 1 3 lj/cut/soft 0.000000 1.000000 0.0 # Hwh Hw +pair_coeff 1 4 lj/cut/soft 0.000000 1.000000 0.0 # Hwh Ow +pair_coeff 2 2 lj/cut/soft 0.155425 3.165500 0.0 # Owh Owh +pair_coeff 2 3 lj/cut/soft 0.000000 1.000000 0.0 # Owh Hw +pair_coeff 2 4 lj/cut/soft 0.155425 3.165500 0.0 # Owh Ow +pair_coeff 3 3 lj/cut/coul/long 0.000000 0.000000 # Hw Hw +pair_coeff 3 4 lj/cut/coul/long 0.000000 0.000000 # Hw Ow +pair_coeff 4 4 lj/cut/coul/long 0.155425 3.165500 # Ow Ow + +# minimize 1.0e-4 1.0e-6 100 1000 +# reset_timestep 0 + +fix SHAKE all shake 0.0001 20 0 b 1 + +neighbor 2.0 bin +# neigh_modify delay 0 every 1 check yes + +timestep 1.0 + +variable TK equal 300.0 +variable PBAR equal 1.0 + +velocity all create ${TK} 12345 + +fix TPSTAT all npt temp ${TK} ${TK} 100 iso ${PBAR} ${PBAR} 1000 + +thermo_style custom step cpu etotal ke pe evdwl ecoul elong temp press vol density +thermo 5000 + +set type 1*2 charge 0.0 + +run 100000 + +reset_timestep 0 + +variable lambda equal ramp(0.0,1.0) + +fix ADAPT all adapt/fep 100000 & + pair lj/cut/soft lambda 1*2 3*4 v_lambda & + after yes + +thermo_style custom step etotal ke pe evdwl ecoul elong temp press density v_lambda + +variable dlambda equal 0.05 + +compute FEP all fep ${TK} & + pair lj/cut/soft lambda 1*2 3*4 v_dlambda & + volume yes + +fix FEP all ave/time 20 4000 100000 c_FEP[*] file fep01-lj.fep + +run 2000000 diff --git a/examples/USER/fep/SPCEhyd/fep01/in-fep01-q.lmp b/examples/USER/fep/SPCEhyd/fep01/in-fep01-q.lmp new file mode 100644 index 0000000000..19efe3a9b1 --- /dev/null +++ b/examples/USER/fep/SPCEhyd/fep01/in-fep01-q.lmp @@ -0,0 +1,78 @@ +# created by fftool + +units real +boundary p p p + +atom_style full +bond_style harmonic +angle_style harmonic + +special_bonds lj/coul 0.0 0.0 0.5 + +# remove hybrid if not necessary +pair_style lj/cut/coul/long 12.0 12.0 +pair_modify tail no +kspace_style pppm 1.0e-5 + +read_data data.lmp + +pair_coeff 1 1 0.000000 1.000000 # Hwh Hwh +pair_coeff 1 2 0.000000 1.000000 # Hwh Owh +pair_coeff 1 3 0.000000 1.000000 # Hwh Hw +pair_coeff 1 4 0.000000 1.000000 # Hwh Ow +pair_coeff 2 2 0.155425 3.165500 # Owh Owh +pair_coeff 2 3 0.000000 1.000000 # Owh Hw +pair_coeff 2 4 0.155425 3.165500 # Owh Ow +pair_coeff 3 3 0.000000 1.000000 # Hw Hw +pair_coeff 3 4 0.000000 1.000000 # Hw Ow +pair_coeff 4 4 0.155425 3.165500 # Ow Ow + +# minimize 1.0e-4 1.0e-6 100 1000 +# reset_timestep 0 + +fix SHAKE all shake 0.0001 20 0 b 1 + +neighbor 2.0 bin +# neigh_modify delay 0 every 1 check yes + +timestep 1.0 + +variable TK equal 300.0 +variable PBAR equal 1.0 + +velocity all create ${TK} 12345 + +fix TPSTAT all npt temp ${TK} ${TK} 100 iso ${PBAR} ${PBAR} 1000 + +thermo_style custom step cpu etotal ke pe evdwl ecoul elong temp press vol density +thermo 5000 + +set type 1*2 charge 0.0 + +run 100000 + +reset_timestep 0 + +variable lambda equal ramp(0.0,1.0) +variable qH equal 0.4238*v_lambda +variable qO equal -0.8476*v_lambda + +fix ADAPT all adapt/fep 100000 & + atom charge 1 v_qH & + atom charge 2 v_qO & + after yes + +thermo_style custom step etotal ke pe evdwl ecoul elong temp press density v_lambda v_qO v_qH + +variable dlambda equal 0.05 +variable dqH equal 0.4238*v_dlambda +variable dqO equal -0.8476*v_dlambda + +compute FEP all fep ${TK} & + atom charge 1 v_dqH & + atom charge 2 v_dqO & + volume yes + +fix FEP all ave/time 20 4000 100000 c_FEP[*] file fep01-q.fep + +run 2000000 diff --git a/examples/USER/fep/SPCEhyd/fep10/fep10-lj.fep b/examples/USER/fep/SPCEhyd/fep10/fep10-lj.fep new file mode 100644 index 0000000000..e08c143542 --- /dev/null +++ b/examples/USER/fep/SPCEhyd/fep10/fep10-lj.fep @@ -0,0 +1,22 @@ +# Time-averaged data for fix FEP +# TimeStep c_FEP[1] c_FEP[2] c_FEP[3] +100000 0.164332 13580.8 17770.4 +200000 0.129134 14470.4 17769.5 +300000 0.113694 14856.5 17741.9 +400000 0.0945078 15445.3 17780.8 +500000 0.0562545 16508.6 17735.1 +600000 0.0490523 16733.2 17757.8 +700000 -0.0236465 19372.6 17801.7 +800000 -0.0387399 19901 17785.3 +900000 -0.0718098 21703.2 17802.6 +1000000 -0.118193 24005.7 17774.1 +1100000 -0.170254 26527.3 17789.9 +1200000 -0.297445 41864.5 17773.6 +1300000 -0.431516 50576.8 17744 +1400000 -0.467669 44025.5 17719.1 +1500000 -0.317884 31440.4 17731.3 +1600000 -0.218939 25792.6 17718.6 +1700000 -0.107798 21322.2 17753.1 +1800000 -0.0458874 19135.5 17706.5 +1900000 -0.015392 18211.1 17744.9 +2000000 -0.00224303 17811.3 17744.3 diff --git a/examples/USER/fep/SPCEhyd/fep10/fep10-lj.out b/examples/USER/fep/SPCEhyd/fep10/fep10-lj.out new file mode 100644 index 0000000000..bc93b1637e --- /dev/null +++ b/examples/USER/fep/SPCEhyd/fep10/fep10-lj.out @@ -0,0 +1,592 @@ +LAMMPS (29 Oct 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (src/lammps/src/comm.cpp:94) + using 1 OpenMP thread(s) per MPI task +Reading data file ... + orthogonal box = (0.0000000 0.0000000 0.0000000) to (29.204526 29.204526 29.204526) + 2 by 2 by 3 MPI processor grid + reading atoms ... + 1800 atoms + scanning bonds ... + 1 = max bonds/atom + scanning angles ... + 1 = max angles/atom + reading bonds ... + 1200 bonds + reading angles ... + 600 angles +Finding 1-2 1-3 1-4 neighbors ... + special bond factors lj: 0 0 0.5 + special bond factors coul: 0 0 0.5 + 2 = max # of 1-2 neighbors + 1 = max # of 1-3 neighbors + 1 = max # of 1-4 neighbors + 2 = max # of special neighbors + special bonds CPU = 0.001 seconds + read_data CPU = 0.033 seconds +Finding SHAKE clusters ... + 0 = # of size 2 clusters + 600 = # of size 3 clusters + 0 = # of size 4 clusters + 0 = # of frozen angles + find clusters CPU = 0.001 seconds +Setting atom values ... + 3 settings made for charge +PPPM initialization ... + using 12-bit tables for long-range coulomb (src/lammps/src/kspace.cpp:339) + G vector (1/distance) = 0.25066829 + grid = 20 20 20 + stencil order = 5 + estimated absolute RMS force accuracy = 0.0030719151 + estimated relative force accuracy = 9.250981e-06 + using double precision FFTW3 + 3d grid and FFT values/proc = 3757 800 +Neighbor list info ... + update every 1 steps, delay 10 steps, check yes + max neighbors/atom: 2000, page size: 100000 + master list distance cutoff = 14 + ghost atom cutoff = 14 + binsize = 7, bins = 5 5 5 + 3 neighbor lists, perpetual/occasional/extra = 3 0 0 + (1) pair lj/cut/coul/long, perpetual, skip from (3) + attributes: half, newton on + pair build: skip + stencil: none + bin: none + (2) pair lj/cut/soft, perpetual, skip from (3) + attributes: half, newton on + pair build: skip + stencil: none + bin: none + (3) neighbor class addition, perpetual + attributes: half, newton on + pair build: half/bin/newton + stencil: half/bin/3d/newton + bin: standard +Setting up Verlet run ... + Unit style : real + Current step : 0 + Time step : 1 +Per MPI rank memory allocation (min/avg/max) = 9.477 | 9.489 | 9.507 Mbytes +Step CPU TotEng KinEng PotEng E_vdwl E_coul E_long Temp Press Volume Density + 0 0 1344.0739 1251.046 93.027896 53.823145 30049.554 -30010.349 300 10284.878 24908.667 0.72058313 + 5000 26.829054 -5531.3966 1246.7673 -6778.164 1430.5662 21645.326 -30297.767 298.97398 343.00879 17939.671 1.0005069 + 10000 60.375331 -5491.2737 1280.0525 -6771.3262 1384.6297 21688.032 -30298.393 306.95574 -300.16806 18083.581 0.99254483 + 15000 94.076576 -5632.0108 1185.0761 -6817.0868 1373.1572 21657.834 -30299.165 284.18046 -509.30462 17853.507 1.0053356 + 20000 126.53271 -5557.9837 1273.3985 -6831.3822 1276.3348 21807.204 -30296.619 305.36012 -1423.8804 17810.87 1.0077422 + 25000 159.26926 -5561.3614 1271.5975 -6832.9588 1454.6816 21523.277 -30299.954 304.92823 245.53675 17839.133 1.0061456 + 30000 191.17362 -5569.1628 1224.4664 -6793.6292 1459.7735 21613.44 -30298.475 293.62623 1166.8411 17466.628 1.0276033 + 35000 225.54672 -5505.5765 1289.9333 -6795.5098 1374.8138 21675.444 -30297.568 309.32515 -277.00241 17845.641 1.0057787 + 40000 259.24748 -5529.8329 1222.5071 -6752.34 1368.482 21714.343 -30299.714 293.15639 -48.777351 17595.097 1.0201004 + 45000 292.69855 -5541.3772 1261.5499 -6802.9272 1371.5857 21681.912 -30295.814 302.51884 -335.65528 17817.714 1.0073551 + 50000 326.91392 -5573.7012 1222.2121 -6795.9133 1327.6979 21746.391 -30299.712 293.08565 -955.31142 17840.793 1.006052 + 55000 361.16613 -5449.164 1267.4384 -6716.6024 1412.1093 21720 -30299.63 303.9309 428.04385 18036.511 0.99513513 + 60000 393.91884 -5546.8912 1190.623 -6737.5143 1330.3252 21806.541 -30300.582 285.51061 -430.37809 17636.451 1.0177085 + 65000 428.22428 -5622.5584 1259.3301 -6881.8885 1509.1614 21478.163 -30299.496 301.98652 975.06072 17605.779 1.0194815 + 70000 462.45487 -5570.183 1218.0018 -6788.1847 1354.4694 21727.024 -30300.976 292.07602 -258.76822 17759.422 1.0106616 + 75000 496.90314 -5627.876 1230.7285 -6858.6045 1476.5693 21530.846 -30297.953 295.12788 553.50896 17749.215 1.0112428 + 80000 531.33879 -5494.2101 1259.607 -6753.8171 1380.283 21711.161 -30298.881 302.05292 211.62464 17627.074 1.0182499 + 85000 564.97941 -5519.5052 1245.7068 -6765.212 1413.8079 21662.496 -30297.084 298.71966 464.57065 17574.97 1.0212686 + 90000 597.70815 -5541.7394 1221.3799 -6763.1193 1438.245 21673.718 -30296.608 292.8861 594.92053 17785.328 1.0091894 + 95000 630.00979 -5506.2477 1292.8624 -6799.1101 1435.7457 21640.887 -30297.759 310.02754 746.74628 17535.628 1.0235599 + 100000 664.4401 -5517.294 1249.4101 -6766.7041 1435.9513 21681.832 -30299.249 299.60772 863.4493 17594.707 1.020123 +Loop time of 664.44 on 12 procs for 100000 steps with 1800 atoms + +Performance: 13.003 ns/day, 1.846 hours/ns, 150.503 timesteps/s +95.1% CPU use with 12 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 354.67 | 370.51 | 389.45 | 60.8 | 55.76 +Bond | 1.2832 | 1.3258 | 1.3684 | 2.3 | 0.20 +Kspace | 122.53 | 141.54 | 156.98 | 99.1 | 21.30 +Neigh | 28.973 | 29.038 | 29.092 | 0.6 | 4.37 +Comm | 42.452 | 45.845 | 51.532 | 54.0 | 6.90 +Output | 0.0013313 | 0.0013703 | 0.0017405 | 0.3 | 0.00 +Modify | 48.441 | 63.192 | 71.445 | 115.5 | 9.51 +Other | | 12.99 | | | 1.95 + +Nlocal: 150.000 ave 163 max 141 min +Histogram: 2 3 2 0 1 0 1 0 2 1 +Nghost: 6146.42 ave 6218 max 6094 min +Histogram: 3 3 0 0 0 0 4 0 1 1 +Neighs: 87791.7 ave 98973 max 80440 min +Histogram: 2 0 3 3 0 2 0 1 0 1 + +Total # of neighbors = 1053500 +Ave neighs/atom = 585.27778 +Ave special neighs/atom = 2.0000000 +Neighbor list builds = 4499 +Dangerous builds = 9 +PPPM initialization ... + using 12-bit tables for long-range coulomb (src/lammps/src/kspace.cpp:339) + G vector (1/distance) = 0.2513327 + grid = 18 18 18 + stencil order = 5 + estimated absolute RMS force accuracy = 0.0034853969 + estimated relative force accuracy = 1.0496169e-05 + using double precision FFTW3 + 3d grid and FFT values/proc = 3328 648 +FEP settings ... + temperature = 300.000000 + tail no + lj/cut/soft lambda 1-2 3-4 +Neighbor list info ... + update every 1 steps, delay 10 steps, check yes + max neighbors/atom: 2000, page size: 100000 + master list distance cutoff = 14 + ghost atom cutoff = 14 + binsize = 7, bins = 4 4 4 + 3 neighbor lists, perpetual/occasional/extra = 3 0 0 + (1) pair lj/cut/coul/long, perpetual, skip from (3) + attributes: half, newton on + pair build: skip + stencil: none + bin: none + (2) pair lj/cut/soft, perpetual, skip from (3) + attributes: half, newton on + pair build: skip + stencil: none + bin: none + (3) neighbor class addition, perpetual + attributes: half, newton on + pair build: half/bin/newton + stencil: half/bin/3d/newton + bin: standard +Setting up Verlet run ... + Unit style : real + Current step : 0 + Time step : 1 +Per MPI rank memory allocation (min/avg/max) = 9.472 | 9.556 | 10.27 Mbytes +Step TotEng KinEng PotEng E_vdwl E_coul E_long Temp Press Density v_lambda + 0 -5517.3031 1249.4101 -6766.7132 1435.9513 21762.036 -30379.462 299.60772 824.0149 1.020123 1 + 5000 -5558.3835 1278.7173 -6837.1007 1334.2641 21745.448 -30378.941 306.63556 -767.88312 1.0259151 0.9975 + 10000 -5536.0094 1281.998 -6818.0074 1400.7848 21722.638 -30382.662 307.42228 286.89597 1.0190965 0.995 + 15000 -5481.3474 1257.9731 -6739.3206 1399.5036 21761.099 -30377.92 301.66113 181.6173 1.0045157 0.9925 + 20000 -5576.3444 1286.5841 -6862.9285 1415.5303 21648.92 -30380.138 308.52201 -153.48635 1.0085717 0.99 + 25000 -5514.2625 1300.4887 -6814.7513 1407.9812 21728.21 -30379.084 311.85634 425.39113 1.018525 0.9875 + 30000 -5547.7134 1261.8317 -6809.5451 1380.2714 21733.843 -30375.593 302.58642 -17.777892 1.0150161 0.985 + 35000 -5477.5978 1281.4333 -6759.0311 1368.0059 21793.139 -30378.187 307.28686 -71.109919 1.0111779 0.9825 + 40000 -5508.6633 1240.2261 -6748.8893 1315.5039 21866.643 -30379.36 297.40539 -903.80351 1.0010855 0.98 + 45000 -5549.0382 1255.1907 -6804.2289 1427.4722 21710.804 -30378.942 300.9939 585.69847 1.0195923 0.9775 + 50000 -5475.0264 1312.3734 -6787.3998 1297.1559 21838.086 -30379.004 314.70627 -1273.035 1.0008054 0.975 + 55000 -5491.7123 1248.8257 -6740.538 1363.4371 21862.536 -30380.311 299.46758 99.128465 1.0180508 0.9725 + 60000 -5609.488 1190.986 -6800.474 1436.0545 21719.642 -30378.046 285.59765 701.33839 1.0113551 0.97 + 65000 -5518.7824 1259.5561 -6778.3385 1358.5984 21791.343 -30380.334 302.04072 -321.61092 1.023541 0.9675 + 70000 -5494.8931 1242.4517 -6737.3448 1305.9749 21912.178 -30379.338 297.9391 -630.02692 1.0139124 0.965 + 75000 -5546.698 1230.9889 -6777.6869 1335.3375 21837.661 -30375.416 295.19033 -852.37291 0.9905341 0.9625 + 80000 -5539.855 1242.8348 -6782.6898 1419.4903 21739.313 -30377.932 298.03096 412.87427 1.01156 0.96 + 85000 -5539.3458 1252.4243 -6791.7701 1324.7893 21793.186 -30381.357 300.33052 -863.18781 1.008687 0.9575 + 90000 -5481.581 1307.2598 -6788.8408 1346.6475 21844.712 -30381.327 313.48005 -408.54122 1.0084955 0.955 + 95000 -5529.9471 1250.3938 -6780.3409 1375.6186 21766.839 -30380.379 299.84361 -208.47283 1.0041284 0.9525 + 100000 -5611.8507 1205.3447 -6817.1954 1432.4671 21667.619 -30379.262 289.04087 575.38545 1.0196531 0.95 + 105000 -5516.1732 1249.0519 -6765.2251 1510.0799 21652.904 -30376.319 299.52182 1595.0527 1.0142387 0.9475 + 110000 -5465.4173 1258.6931 -6724.1103 1349.0925 21850.539 -30376.601 301.83377 -603.2308 0.98281583 0.945 + 115000 -5551.5788 1260.591 -6812.1698 1512.1661 21629.589 -30379.324 302.28889 1408.8013 1.0119216 0.9425 + 120000 -5507.219 1271.1602 -6778.3792 1287.9978 21860.629 -30381.307 304.82338 -1063.0529 1.0271704 0.94 + 125000 -5537.13 1258.8105 -6795.9405 1382.8613 21762.733 -30377.383 301.86193 -158.02523 1.0108426 0.9375 + 130000 -5549.0946 1266.6866 -6815.7812 1372.5433 21748.74 -30377.189 303.75061 -643.27667 0.99432531 0.935 + 135000 -5573.4448 1286.5394 -6859.9842 1446.9417 21606.202 -30379.095 308.51131 579.59371 1.0183103 0.9325 + 140000 -5486.2607 1218.2948 -6704.5555 1391.8317 21866.056 -30379.705 292.14629 239.66325 1.0017938 0.93 + 145000 -5528.1392 1237.3437 -6765.4829 1343.2831 21851.228 -30378.61 296.71421 -443.27605 1.0094072 0.9275 + 150000 -5524.9227 1258.9283 -6783.851 1371.8204 21782.513 -30377.396 301.89018 -100.72793 1.0162581 0.925 + 155000 -5559.5874 1249.0683 -6808.6557 1298.8221 21822.678 -30380.108 299.52576 -1283.9404 1.0098295 0.9225 + 160000 -5574.1599 1259.0423 -6833.2022 1386.5012 21768.271 -30378.898 301.91751 -274.81981 1.0036715 0.92 + 165000 -5558.2274 1268.7313 -6826.9587 1366.6284 21720.826 -30379.54 304.24093 -723.57468 1.0031713 0.9175 + 170000 -5558.0458 1243.1304 -6801.1762 1359.415 21794.026 -30377.538 298.10185 -284.08503 1.0125849 0.915 + 175000 -5557.9939 1250.5207 -6808.5146 1344.1211 21819.765 -30375.934 299.87403 -742.04372 1.008519 0.9125 + 180000 -5562.2438 1245.4126 -6807.6564 1442.5917 21688.098 -30381.444 298.64912 834.40431 1.0283372 0.91 + 185000 -5494.5329 1238.0904 -6732.6233 1374.4489 21815.526 -30379.394 296.89325 -127.96552 1.0129962 0.9075 + 190000 -5512.3632 1282.3603 -6794.7235 1331.0347 21826.906 -30381.177 307.50916 -975.48387 0.99030439 0.905 + 195000 -5555.5139 1268.6099 -6824.1238 1579.0357 21509.544 -30380.501 304.21181 2298.8088 1.0238224 0.9025 + 200000 -5561.6068 1235.0747 -6796.6815 1341.7814 21791.238 -30377.819 296.1701 -569.19717 1.0181737 0.9 + 205000 -5408.9255 1277.421 -6686.3465 1384.0288 21847.016 -30376.919 306.3247 306.84932 1.0058379 0.8975 + 210000 -5574.4209 1249.0534 -6823.4743 1390.0917 21708.289 -30375.252 299.52217 -137.92578 1.020659 0.895 + 215000 -5533.5343 1256.0469 -6789.5812 1397.7965 21759.307 -30379.639 301.19922 -167.33553 0.99448086 0.8925 + 220000 -5525.1497 1272.4337 -6797.5834 1349.1947 21798.064 -30375.86 305.12875 -636.29714 1.0040906 0.89 + 225000 -5504.1279 1280.7318 -6784.8598 1373.0777 21811.329 -30381.594 307.11864 -109.03674 1.0126578 0.8875 + 230000 -5556.0721 1227.0604 -6783.1325 1460.4513 21700.918 -30378.062 294.24828 1031.1913 1.0144785 0.885 + 235000 -5579.4548 1223.0151 -6802.47 1383.0244 21742.747 -30378.065 293.27823 -444.73244 0.99883895 0.8825 + 240000 -5478.8969 1256.3704 -6735.2673 1379.5164 21844.649 -30379.291 301.2768 134.8168 1.0086341 0.88 + 245000 -5473.7115 1247.7991 -6721.5106 1377.2785 21845.407 -30379.554 299.22139 152.12047 1.0028516 0.8775 + 250000 -5511.3331 1252.6671 -6764.0002 1391.242 21767.929 -30377.767 300.38874 42.815705 1.0071501 0.875 + 255000 -5527.2234 1305.1549 -6832.3783 1450.6004 21675.372 -30380.236 312.97529 568.31787 1.0194011 0.8725 + 260000 -5599.9408 1267.0163 -6866.9571 1414.5319 21681.001 -30380.447 303.82967 -296.86382 0.99844118 0.87 + 265000 -5609.1575 1202.8066 -6811.9641 1422.9683 21708.562 -30377.312 288.43224 396.28769 1.0282155 0.8675 + 270000 -5514.4973 1238.2917 -6752.789 1478.2795 21711.681 -30377.388 296.94154 987.01375 0.99825963 0.865 + 275000 -5511.4623 1255.7444 -6767.2068 1360.2011 21806.795 -30378.728 301.12669 -323.15359 1.0062462 0.8625 + 280000 -5552.1793 1259.5205 -6811.6998 1349.3495 21816.208 -30382.584 302.03219 -349.89702 1.0176732 0.86 + 285000 -5533.2625 1244.4885 -6777.751 1329.1246 21846.62 -30381.164 298.42752 -459.22534 1.0226658 0.8575 + 290000 -5567.8243 1241.1772 -6809.0015 1399.9451 21749.571 -30379.349 297.63346 22.543027 1.0110614 0.855 + 295000 -5557.5037 1238.0302 -6795.5339 1338.4732 21820.049 -30378.884 296.87882 -618.59847 1.0068859 0.8525 + 300000 -5489.886 1256.7172 -6746.6033 1363.8733 21840.359 -30378.014 301.35996 -70.896612 1.0126564 0.85 + 305000 -5572.9241 1240.321 -6813.245 1460.9673 21642.12 -30380.518 297.42815 481.8838 1.0057691 0.8475 + 310000 -5488.162 1258.2317 -6746.3936 1445.6895 21758.579 -30378.851 301.72313 872.39978 1.0085289 0.845 + 315000 -5522.8931 1237.6936 -6760.5868 1394.4464 21761.506 -30375.489 296.79811 209.47881 1.0172465 0.8425 + 320000 -5536.6098 1282.2086 -6818.8184 1354.2609 21746.18 -30376.821 307.47278 -800.05482 1.001315 0.84 + 325000 -5531.7735 1280.9795 -6812.753 1415.0442 21718.513 -30376.752 307.17804 140.19074 1.0033268 0.8375 + 330000 -5610.5654 1255.9186 -6866.4839 1439.716 21627.111 -30376.917 301.16845 217.00726 1.0106958 0.835 + 335000 -5544.1968 1262.8047 -6807.0015 1411.1851 21727.544 -30379.348 302.81973 -110.66815 0.99018265 0.8325 + 340000 -5475.3641 1226.3203 -6701.6844 1319.0089 21921.241 -30376.884 294.07079 -282.11126 1.0213371 0.83 + 345000 -5553.4383 1194.8379 -6748.2762 1355.9404 21856.676 -30381.74 286.52133 -362.738 1.0061755 0.8275 + 350000 -5584.6493 1235.7595 -6820.4088 1352.8128 21761.048 -30380.806 296.33432 -874.17129 0.9931264 0.825 + 355000 -5621.2604 1217.1919 -6838.4523 1372.941 21733.863 -30375.651 291.88181 -584.44984 0.99879817 0.8225 + 360000 -5615.0717 1231.0617 -6846.1334 1429.3525 21680.466 -30380.519 295.20778 330.32191 1.0138088 0.82 + 365000 -5589.4166 1229.8012 -6819.2178 1445.1399 21656.389 -30379.255 294.90552 421.5138 1.0090829 0.8175 + 370000 -5651.5079 1193.437 -6844.9449 1435.9396 21645.199 -30380.798 286.18541 256.92768 1.0192016 0.815 + 375000 -5552.4603 1258.4199 -6810.8802 1378.5865 21748.986 -30378.675 301.76827 -383.52078 0.99353397 0.8125 + 380000 -5504.5385 1251.8059 -6756.3444 1356.5769 21819.021 -30380.364 300.18223 -269.24879 1.0038731 0.81 + 385000 -5555.4347 1224.5956 -6780.0303 1375.7528 21743.978 -30382.275 293.65722 150.61285 1.033997 0.8075 + 390000 -5476.7978 1301.5154 -6778.3132 1470.6852 21684.408 -30378.178 312.10253 1146.8841 1.0219575 0.805 + 395000 -5396.8713 1298.4198 -6695.2912 1317.1078 21924.316 -30374.204 311.36022 -823.77541 0.98553613 0.8025 + 400000 -5629.6891 1238.2653 -6867.9544 1428.4096 21673.519 -30377.557 296.9352 206.26016 1.0115683 0.8 + 405000 -5428.7589 1278.424 -6707.1828 1331.4932 21893.287 -30377.714 306.56523 -353.63082 1.0124706 0.7975 + 410000 -5507.5696 1237.8639 -6745.4335 1298.4114 21901.188 -30379.626 296.83895 -973.92318 1.0037112 0.795 + 415000 -5484.2415 1270.2855 -6754.527 1441.5793 21749.552 -30375.284 304.61361 840.14191 1.0121738 0.7925 + 420000 -5495.6378 1258.7861 -6754.4239 1353.3243 21849.806 -30381.941 301.85607 59.331351 1.0246191 0.79 + 425000 -5552.4141 1228.8708 -6781.2849 1317.4814 21842.189 -30380.175 294.68241 -836.50476 1.0135725 0.7875 + 430000 -5477.5725 1254.5002 -6732.0727 1367.7274 21841.064 -30380.543 300.82831 88.637178 1.011139 0.785 + 435000 -5476.7143 1264.4344 -6741.1487 1375.6575 21790.072 -30378.636 303.21053 2.6220187 1.0159663 0.7825 + 440000 -5565.7547 1202.2639 -6768.0186 1376.2632 21836.685 -30377.795 288.30208 -159.4843 1.0101374 0.78 + 445000 -5482.073 1263.9755 -6746.0485 1298.6972 21912.114 -30380.1 303.10049 -837.55689 1.009925 0.7775 + 450000 -5471.1641 1270.7405 -6741.9046 1327.0686 21843.353 -30380.6 304.72273 -812.71837 1.0073281 0.775 + 455000 -5575.8919 1237.6557 -6813.5477 1403.2693 21726.183 -30378.18 296.78902 -58.778086 1.0052848 0.7725 + 460000 -5483.7083 1255.8916 -6739.5999 1386.2234 21823.459 -30378.531 301.16198 150.3882 1.0047493 0.77 + 465000 -5605.7393 1229.0621 -6834.8014 1326.0846 21768.021 -30377.81 294.72828 -577.61705 1.038623 0.7675 + 470000 -5550.9894 1229.8014 -6780.7909 1437.1655 21776.095 -30380.267 294.90557 774.11419 1.0140737 0.765 + 475000 -5523.3444 1300.4873 -6823.8318 1327.0338 21792.331 -30381.352 311.856 -970.02145 1.0142989 0.7625 + 480000 -5531.1145 1240.4904 -6771.6049 1371.1764 21817.322 -30375.833 297.46877 232.38091 1.0354183 0.76 + 485000 -5586.0909 1228.566 -6814.6569 1331.4521 21783.37 -30375.359 294.60932 -830.78206 1.0182576 0.7575 + 490000 -5529.1137 1267.8457 -6796.9594 1413.3701 21752.62 -30377.127 304.02856 442.05038 1.0159886 0.755 + 495000 -5555.6588 1216.5915 -6772.2503 1419.8591 21749.535 -30380.057 291.73784 304.69606 1.0095655 0.7525 + 500000 -5621.0389 1231.8288 -6852.8678 1474.6487 21612.415 -30378.168 295.39174 323.36029 1.0077507 0.75 + 505000 -5546.0577 1251.7571 -6797.8148 1367.67 21795.98 -30376.544 300.17053 -455.53831 1.0072369 0.7475 + 510000 -5582.9058 1221.373 -6804.2787 1350.5321 21782.21 -30381.612 292.88444 -772.87812 1.0034727 0.745 + 515000 -5541.3247 1255.4485 -6796.7732 1414.1242 21706.658 -30381.741 301.05573 2.928327 0.99672686 0.7425 + 520000 -5508.7799 1259.6743 -6768.4541 1380.3991 21741.143 -30380.532 302.06906 -203.6546 1.0136344 0.74 + 525000 -5539.6518 1197.3021 -6736.9538 1404.1918 21779.603 -30380.69 287.11225 145.06485 1.003219 0.7375 + 530000 -5624.3435 1217.1528 -6841.4963 1452.7693 21662.529 -30380.236 291.87243 638.82307 1.0215612 0.735 + 535000 -5529.8222 1254.4932 -6784.3155 1406.2098 21740.849 -30378.658 300.82665 94.641198 1.0065968 0.7325 + 540000 -5568.7637 1261.6812 -6830.4449 1387.5129 21733.157 -30379.551 302.55032 -405.87495 1.0104021 0.73 + 545000 -5524.6427 1291.6553 -6816.298 1444.7739 21695.48 -30380.734 309.73809 823.06856 1.0215824 0.7275 + 550000 -5545.8811 1294.844 -6840.725 1393.6405 21689.154 -30381.254 310.50273 -155.25174 1.0231418 0.725 + 555000 -5524.836 1234.4479 -6759.2838 1356.4213 21802.616 -30369.49 296.01978 -368.99982 1.0143158 0.7225 + 560000 -5604.2588 1246.9449 -6851.2037 1434.9571 21646.273 -30381.555 299.01658 256.97524 1.0137294 0.72 + 565000 -5568.3972 1224.1467 -6792.5439 1448.4206 21684.449 -30378.954 293.54958 556.78887 1.0032844 0.7175 + 570000 -5594.4587 1230.6686 -6825.1274 1435.707 21681.561 -30379.685 295.11353 473.82707 1.0198226 0.715 + 575000 -5488.4667 1277.4643 -6765.931 1404.5892 21784.485 -30376.117 306.3351 362.86154 1.0003658 0.7125 + 580000 -5547.1998 1264.7062 -6811.906 1393.9582 21720.572 -30378.614 303.27571 -137.91377 1.0172504 0.71 + 585000 -5606.6441 1252.6018 -6859.2459 1419.0008 21663.205 -30378.365 300.37309 66.460552 1.021903 0.7075 + 590000 -5512.7176 1269.3615 -6782.0791 1408.1893 21757.063 -30379.393 304.39205 644.37249 1.0364827 0.705 + 595000 -5608.6134 1228.9868 -6837.6002 1433.5598 21649.995 -30378.63 294.71023 32.723185 1.0054794 0.7025 + 600000 -5552.3005 1223.3003 -6775.6008 1369.5223 21840.904 -30379.11 293.34662 -395.91281 0.99377694 0.7 + 605000 -5536.8925 1270.6202 -6807.5127 1323.5731 21800.775 -30378.632 304.69388 -977.46567 1.0071163 0.6975 + 610000 -5464.7705 1275.0987 -6739.8692 1367.5489 21824.895 -30377.252 305.76782 -387.65672 0.99109537 0.695 + 615000 -5609.4125 1262.2387 -6871.6513 1408.4333 21658.685 -30377.411 302.68401 22.836181 1.0270818 0.6925 + 620000 -5531.8104 1274.3728 -6806.1831 1377.4975 21766.69 -30379.63 305.59376 -285.65697 1.0117261 0.69 + 625000 -5474.4189 1278.1649 -6752.5838 1333.798 21868.283 -30379.65 306.5031 -521.91598 1.0102571 0.6875 + 630000 -5538.9642 1225.28 -6764.2442 1389.7032 21773.906 -30378.718 293.82133 254.50002 1.0208426 0.685 + 635000 -5508.4261 1280.6282 -6789.0543 1401.0525 21750.972 -30379.98 307.09381 234.86257 1.0110397 0.6825 + 640000 -5540.8089 1244.3064 -6785.1153 1417.4663 21726.439 -30376.915 298.38384 150.36017 1.0001641 0.68 + 645000 -5515.8042 1236.468 -6752.2723 1402.2548 21790.109 -30379.114 296.50422 531.96132 1.020622 0.6775 + 650000 -5462.1993 1257.9462 -6720.1454 1268.863 21936.688 -30379.734 301.65466 -1017.4425 1.0216886 0.675 + 655000 -5486.7934 1301.7279 -6788.5213 1355.2019 21810.937 -30378.882 312.1535 -617.30085 0.99817188 0.6725 + 660000 -5534.9148 1263.11 -6798.0248 1378.272 21775.106 -30376.702 302.89295 -223.77151 1.0075378 0.67 + 665000 -5581.7219 1234.0612 -6815.7831 1428.1581 21716.123 -30378.571 295.92706 44.290152 0.99842109 0.6675 + 670000 -5488.6526 1269.5454 -6758.198 1367.3963 21828.425 -30380.546 304.43615 -348.95619 1.0000298 0.665 + 675000 -5465.914 1263.3905 -6729.3045 1336.7565 21880.293 -30376.633 302.96021 -485.33842 1.003274 0.6625 + 680000 -5472.168 1264.3649 -6736.533 1448.9293 21749.791 -30379.242 303.19388 831.6918 1.0029436 0.66 + 685000 -5520.395 1250.1857 -6770.5807 1401.6986 21805.009 -30376.553 299.79371 -41.384636 0.99397705 0.6575 + 690000 -5480.0792 1289.9138 -6769.993 1420.5975 21783.734 -30378.473 309.32048 460.10945 1.0106925 0.655 + 695000 -5589.3538 1224.5521 -6813.9059 1451.0824 21702.734 -30376.351 293.64678 738.57116 1.0153307 0.6525 + 700000 -5472.8453 1308.7177 -6781.563 1423.187 21741.303 -30378.197 313.82963 612.6714 1.0166905 0.65 + 705000 -5569.7342 1255.2584 -6824.9926 1389.4201 21710.248 -30380.297 301.01013 -172.7884 1.0148878 0.6475 + 710000 -5596.1426 1238.2633 -6834.4059 1357.9019 21745.397 -30380.137 296.93471 -538.38313 1.0209937 0.645 + 715000 -5507.4206 1239.2393 -6746.6599 1403.4807 21781.313 -30378.667 297.16877 248.26197 1.0116725 0.6425 + 720000 -5497.1275 1287.2767 -6784.4042 1414.7207 21756.731 -30379.238 308.68811 341.59938 1.0131454 0.64 + 725000 -5496.5836 1286.0241 -6782.6077 1399.7439 21764.503 -30379.835 308.38773 -62.342241 0.99678722 0.6375 + 730000 -5536.977 1243.2012 -6780.1782 1317.0319 21827.887 -30379.541 298.11883 -1280.4499 0.99058396 0.635 + 735000 -5549.9255 1278.5327 -6828.4582 1466.7074 21634.42 -30377.429 306.5913 700.98665 1.0145657 0.6325 + 740000 -5496.3831 1282.2407 -6778.6238 1324.4366 21812.16 -30377.5 307.48047 -793.87783 1.0141821 0.63 + 745000 -5500.4874 1275.7087 -6776.1962 1310.7145 21869.953 -30378.837 305.91411 -886.8993 1.0111339 0.6275 + 750000 -5500.9475 1276.7305 -6777.678 1395.7077 21776.63 -30380.09 306.15913 192.19564 1.0108423 0.625 + 755000 -5551.394 1240.5899 -6791.9839 1454.451 21659.096 -30378.918 297.49265 498.47384 1.0007052 0.6225 + 760000 -5528.5295 1245.9619 -6774.4915 1447.3278 21723.641 -30377.888 298.78085 515.39703 1.005014 0.62 + 765000 -5475.7763 1268.2993 -6744.0756 1335.9202 21904.289 -30375.821 304.13733 -285.98001 1.0140331 0.6175 + 770000 -5571.7813 1239.3782 -6811.1595 1445.9375 21665.43 -30380.469 297.20208 428.18308 1.0025023 0.615 + 775000 -5501.6766 1254.1992 -6755.8758 1320.6755 21872.392 -30377.78 300.75613 -619.04607 1.0168732 0.6125 + 780000 -5546.2901 1244.0883 -6790.3785 1387.3647 21762.273 -30378.875 298.33156 -132.28122 1.0042433 0.61 + 785000 -5472.3472 1281.5277 -6753.8748 1347.5012 21865.731 -30381.351 307.30949 -155.27002 1.0077831 0.6075 + 790000 -5537.4018 1231.8299 -6769.2317 1348.3549 21856.185 -30381.91 295.392 -210.503 1.0194699 0.605 + 795000 -5566.3088 1261.8098 -6828.1186 1449.2851 21688.44 -30377.026 302.58115 1001.4476 1.0322702 0.6025 + 800000 -5471.8234 1303.4049 -6775.2283 1398.6426 21772.653 -30379.166 312.55564 217.87426 1.0135949 0.6 + 805000 -5586.6474 1220.6453 -6807.2927 1409.1848 21676.401 -30378.777 292.70993 241.46493 1.0243302 0.5975 + 810000 -5612.0552 1215.6221 -6827.6773 1363.5428 21759.362 -30378.853 291.50537 -711.05271 1.0076206 0.595 + 815000 -5584.3197 1224.7552 -6809.0749 1469.9794 21677.26 -30382.578 293.69549 789.61759 1.0091541 0.5925 + 820000 -5552.9536 1270.928 -6823.8816 1395.5282 21729.655 -30380.525 304.76769 132.06866 1.0153299 0.59 + 825000 -5524.6445 1255.7384 -6780.3829 1467.1114 21702.335 -30380.146 301.12523 1094.8403 1.0124363 0.5875 + 830000 -5519.9763 1283.1177 -6803.094 1359.1241 21781.756 -30378.661 307.69078 -269.55156 1.011247 0.585 + 835000 -5574.5608 1268.1186 -6842.6794 1453.1708 21651.026 -30378.923 304.09401 440.04367 1.0063498 0.5825 + 840000 -5593.1298 1263.5752 -6856.705 1437.8518 21679.6 -30380.449 303.00449 786.15751 1.0315038 0.58 + 845000 -5474.6027 1278.2525 -6752.8552 1345.4766 21857.636 -30379.511 306.52411 -375.02983 1.0081416 0.5775 + 850000 -5421.5672 1307.2683 -6728.8355 1354.7412 21860.822 -30377.514 313.48207 -176.84475 1.0001495 0.575 + 855000 -5523.8453 1251.0854 -6774.9307 1424.8602 21729.911 -30380.62 300.00944 567.62922 1.0185387 0.5725 + 860000 -5540.1719 1247.9292 -6788.1011 1338.1786 21819.946 -30380.808 299.25259 -980.8771 0.98328196 0.57 + 865000 -5500.9874 1294.2799 -6795.2673 1365.5005 21798.964 -30380.394 310.36746 -312.14623 1.0038906 0.5675 + 870000 -5513.3883 1253.1896 -6766.5779 1403.6774 21770.72 -30379.858 300.51403 -249.64253 0.98661096 0.565 + 875000 -5577.3005 1213.9579 -6791.2584 1373.5687 21767.892 -30379.213 291.10631 -32.675529 1.0214726 0.5625 + 880000 -5520.6187 1235.4014 -6756.0201 1414.0406 21755.825 -30377.065 296.24843 539.88708 1.0112853 0.56 + 885000 -5506.5962 1242.8121 -6749.4082 1313.1296 21895.863 -30377.787 298.02552 -810.01171 1.0046804 0.5575 + 890000 -5574.2541 1264.4641 -6838.7182 1462.9071 21672.919 -30379.655 303.21767 725.59742 1.0127452 0.555 + 895000 -5552.393 1266.4737 -6818.8667 1451.4529 21687.174 -30377.823 303.69956 515.3533 1.0046976 0.5525 + 900000 -5493.546 1268.8835 -6762.4295 1390.089 21773.231 -30379.297 304.27743 323.6865 1.0191801 0.55 + 905000 -5568.8411 1251.4696 -6820.3107 1415.2346 21698.664 -30379.466 300.10159 142.08451 1.0164587 0.5475 + 910000 -5472.0573 1255.734 -6727.7913 1371.5728 21814.959 -30376.67 301.12419 71.603808 1.0152894 0.545 + 915000 -5546.7758 1266.2377 -6813.0135 1394.2902 21729.23 -30378.982 303.64295 -277.04292 0.99838613 0.5425 + 920000 -5485.1925 1260.6795 -6745.8719 1335.4147 21840.054 -30376.213 302.3101 -706.48845 0.99982968 0.54 + 925000 -5569.2581 1268.7997 -6838.0578 1425.5561 21688.585 -30376.69 304.25734 440.61245 1.0276647 0.5375 + 930000 -5436.2 1309.4362 -6745.6361 1360.6477 21779.291 -30375.596 314.00193 -153.76175 1.0060773 0.535 + 935000 -5511.9611 1283.1051 -6795.0662 1472.8015 21686.056 -30378.591 307.68775 830.7404 1.0035227 0.5325 + 940000 -5525.2952 1250.2261 -6775.5212 1382.7242 21797.489 -30378.43 299.80339 411.53116 1.0379394 0.53 + 945000 -5479.6309 1302.2216 -6781.8525 1392.5574 21734.537 -30380.406 312.27189 -109.18487 1.0077314 0.5275 + 950000 -5432.2038 1264.9369 -6697.1407 1350.3293 21930.966 -30376.591 303.33102 181.11895 1.0110981 0.525 + 955000 -5560.8112 1217.4105 -6778.2217 1444.1975 21748.915 -30379.038 291.93423 713.37875 1.0089567 0.5225 + 960000 -5522.7277 1295.5111 -6818.2388 1463.1361 21672.625 -30375.593 310.66272 497.12398 0.99359159 0.52 + 965000 -5476.6514 1255.7119 -6732.3633 1454.9215 21761.733 -30377.381 301.11888 891.7686 1.0003201 0.5175 + 970000 -5554.9486 1260.713 -6815.6616 1444.2112 21699.023 -30376.601 302.31814 734.59668 1.0255112 0.515 + 975000 -5537.0867 1297.5833 -6834.6701 1454.7241 21625.665 -30378.948 311.15963 884.84024 1.026448 0.5125 + 980000 -5546.2148 1224.275 -6770.4898 1333.2213 21873.175 -30378.024 293.58034 -493.4515 1.012591 0.51 + 985000 -5520.9359 1217.3655 -6738.3014 1343.6097 21870.6 -30375.709 291.92345 -263.92883 1.0149279 0.5075 + 990000 -5592.7447 1215.8757 -6808.6205 1440.2464 21666.537 -30374.994 291.5662 587.54768 1.0234674 0.505 + 995000 -5533.0301 1262.5102 -6795.5404 1279.6793 21920.053 -30380.017 302.74912 -1341.8752 1.0002809 0.5025 + 1000000 -5551.0164 1238.3305 -6789.3469 1391.9315 21798.137 -30377.817 296.95085 -113.77494 1.0094761 0.5 + 1005000 -5549.2261 1237.9081 -6787.1342 1320.049 21844.294 -30382.381 296.84955 -634.07433 1.0208379 0.4975 + 1010000 -5529.9238 1239.8968 -6769.8205 1389.9921 21747.565 -30379.146 297.32642 -0.90599205 0.99978793 0.495 + 1015000 -5489.9865 1252.1385 -6742.1251 1398.0166 21799.294 -30379.371 300.262 241.56608 1.0076692 0.4925 + 1020000 -5576.0301 1272.5378 -6848.5679 1416.7164 21667.174 -30378.805 305.15372 -33.86377 0.99969748 0.49 + 1025000 -5591.0562 1246.2942 -6837.3504 1438.919 21638.67 -30378.781 298.86053 482.55574 1.0162828 0.4875 + 1030000 -5548.1877 1217.7539 -6765.9416 1362.9837 21806.283 -30381.163 292.01659 -427.99018 0.99163938 0.485 + 1035000 -5522.7032 1310.9673 -6833.6704 1454.9127 21653.869 -30379.885 314.36908 738.25531 1.0236085 0.4825 + 1040000 -5589.4653 1211.212 -6800.6774 1396.8117 21735.372 -30375.228 290.44785 -167.16934 1.0043731 0.48 + 1045000 -5584.8632 1213.0589 -6797.9222 1345.4118 21771.193 -30380.486 290.89073 -797.43246 1.0092941 0.4775 + 1050000 -5507.4502 1224.5435 -6731.9938 1326.4546 21912.141 -30377.193 293.64473 -497.19203 1.0032709 0.475 + 1055000 -5493.9189 1230.8729 -6724.7919 1385.1499 21849.807 -30375.98 295.16252 218.46916 0.99861109 0.4725 + 1060000 -5555.3241 1241.6789 -6797.003 1384.3354 21771.345 -30377.318 297.75379 -237.59628 0.99873701 0.47 + 1065000 -5550.2811 1282.7371 -6833.0182 1548.4595 21554.165 -30376.611 307.59951 1670.18 1.0096302 0.4675 + 1070000 -5535.527 1254.4489 -6789.9759 1438.3778 21690.349 -30376.448 300.81601 491.9271 1.0179693 0.465 + 1075000 -5537.4925 1273.5683 -6811.0609 1437.8103 21688.987 -30379.423 305.40085 802.67013 1.0289308 0.4625 + 1080000 -5570.3365 1249.4877 -6819.8242 1420.2567 21697.63 -30374.946 299.62632 180.96347 1.0118406 0.46 + 1085000 -5585.0345 1210.3922 -6795.4267 1467.9551 21664.251 -30378.174 290.25124 794.72418 1.0055281 0.4575 + 1090000 -5508.2604 1219.7555 -6728.0159 1352.4211 21869.903 -30377.463 292.49657 -357.12347 1.0108237 0.455 + 1095000 -5518.5734 1279.9107 -6798.4841 1448.7068 21686.898 -30379.086 306.92175 722.82217 1.0109523 0.4525 + 1100000 -5525.8416 1250.687 -6776.5287 1414.8834 21735.488 -30379.591 299.91393 197.94954 1.0154492 0.45 + 1105000 -5521.8723 1246.6246 -6768.497 1401.1347 21814.8 -30376.034 298.93976 117.88115 0.99889954 0.4475 + 1110000 -5554.5946 1270.4005 -6824.9951 1365.948 21741.545 -30380.13 304.64121 -662.90851 0.99998764 0.445 + 1115000 -5545.4198 1250.2667 -6795.6866 1348.0742 21791.444 -30381.204 299.81314 -724.2404 1.0103995 0.4425 + 1120000 -5559.3322 1197.8416 -6757.1738 1355.7143 21850.726 -30380.054 287.24162 -181.20192 1.0173846 0.44 + 1125000 -5542.1037 1234.4106 -6776.5142 1379.6262 21797.166 -30380.781 296.01084 -507.86039 0.99365362 0.4375 + 1130000 -5437.0102 1287.5891 -6724.5993 1282.7916 21954.329 -30381.096 308.76302 -1126.2965 0.99901743 0.435 + 1135000 -5538.9176 1238.9073 -6777.8249 1364.7683 21805.703 -30379.16 297.08916 -680.37057 0.9996137 0.4325 + 1140000 -5570.4962 1213.8459 -6784.3421 1340.606 21805.103 -30379.425 291.07945 -596.29765 1.0134925 0.43 + 1145000 -5547.5904 1230.6761 -6778.2665 1420.8752 21738.324 -30378.751 295.11533 246.41006 1.006487 0.4275 + 1150000 -5548.2808 1213.8732 -6762.154 1421.7664 21766.257 -30379.185 291.08599 570.07885 1.0218796 0.425 + 1155000 -5521.4144 1267.3636 -6788.778 1398.1887 21770.241 -30379.549 303.91295 2.9983687 1.0076686 0.4225 + 1160000 -5534.9879 1244.8652 -6779.8532 1440.4749 21715.397 -30378.941 298.51786 679.4149 1.0207784 0.42 + 1165000 -5484.3721 1244.6856 -6729.0577 1417.7678 21735.834 -30377.46 298.4748 629.57345 1.0215953 0.4175 + 1170000 -5524.3406 1277.2058 -6801.5464 1389.395 21753.644 -30381.276 306.27312 85.448076 1.0175567 0.415 + 1175000 -5573.4085 1245.0919 -6818.5004 1464.992 21672.608 -30376.328 298.57221 727.70962 1.0151094 0.4125 + 1180000 -5567.9892 1220.0499 -6788.039 1355.1799 21757.217 -30379.53 292.56716 -589.39342 1.0129301 0.41 + 1185000 -5493.5883 1254.9729 -6748.5612 1378.5484 21787.909 -30376.087 300.94166 75.165141 1.0184138 0.4075 + 1190000 -5479.2948 1260.2407 -6739.5356 1356.4075 21814.922 -30377.304 302.2049 -460.01063 1.0009331 0.405 + 1195000 -5457.6359 1238.4178 -6696.0537 1364.6441 21884.995 -30379.875 296.97177 3.5756347 1.0001974 0.4025 + 1200000 -5469.0842 1241.7112 -6710.7954 1406.1586 21802.207 -30379.157 297.76153 438.48868 1.0195469 0.4 + 1205000 -5549.5458 1240.8355 -6790.3813 1358.959 21808.604 -30379.225 297.55153 -594.51594 1.0074724 0.3975 + 1210000 -5585.1155 1238.9458 -6824.0613 1456.1794 21691.63 -30379.186 297.09838 1167.2584 1.0397342 0.395 + 1215000 -5533.2061 1246.371 -6779.5771 1439.4805 21720.932 -30378.201 298.87894 526.47308 1.0105713 0.3925 + 1220000 -5543.7123 1246.3605 -6790.0728 1445.4823 21714.058 -30379.216 298.87643 677.38399 1.0072857 0.39 + 1225000 -5474.7986 1269.4558 -6744.2544 1364.5803 21848.038 -30377.16 304.41467 -87.090524 1.0086417 0.3875 + 1230000 -5541.2817 1308.326 -6849.6077 1415.265 21695.861 -30381.038 313.73572 71.444446 1.0192522 0.385 + 1235000 -5542.6225 1244.4364 -6787.0589 1381.0607 21784.705 -30380.057 298.41502 214.57658 1.0258239 0.3825 + 1240000 -5555.7484 1274.2793 -6830.0277 1489.2232 21617.829 -30378.278 305.57134 876.45857 1.008025 0.38 + 1245000 -5512.6649 1274.3059 -6786.9708 1382.4273 21743.608 -30378.966 305.57772 -122.08537 1.020499 0.3775 + 1250000 -5589.2861 1219.0535 -6808.3396 1360.303 21791.212 -30380.132 292.32822 -128.28745 1.0295553 0.375 + 1255000 -5535.2523 1239.0819 -6774.3342 1352.7449 21789.218 -30378.876 297.13101 -679.25947 0.99401271 0.3725 + 1260000 -5570.865 1227.2408 -6798.1058 1427.1345 21699.292 -30378.045 294.29153 327.24356 1.006499 0.37 + 1265000 -5471.9137 1290.8466 -6762.7604 1381.3429 21818.652 -30379.538 309.54417 14.98415 1.0111953 0.3675 + 1270000 -5450.4791 1305.5008 -6755.98 1370.0496 21830.174 -30379.524 313.05824 -293.10433 0.99617081 0.365 + 1275000 -5471.3926 1284.8849 -6756.2775 1367.9952 21845.798 -30374.717 308.11454 -307.10586 0.99444636 0.3625 + 1280000 -5520.0507 1269.7599 -6789.8106 1386.8953 21790.683 -30379.921 304.48759 10.244879 1.0094059 0.36 + 1285000 -5595.952 1204.1409 -6800.0929 1398.6057 21718.538 -30379.702 288.7522 -276.8686 1.006966 0.3575 + 1290000 -5486.8122 1246.1647 -6732.9769 1363.6203 21823.272 -30379.685 298.82947 -464.78674 0.99243587 0.355 + 1295000 -5497.4179 1237.867 -6735.2849 1346.8425 21891.759 -30377.608 296.83969 -140.11806 1.0151073 0.3525 + 1300000 -5639.1001 1243.6172 -6882.7173 1457.7891 21616.148 -30378.09 298.21857 342.48703 1.009843 0.35 + 1305000 -5556.8881 1221.543 -6778.431 1335.1406 21849.077 -30378.112 292.9252 -716.76985 1.0013082 0.3475 + 1310000 -5442.7105 1258.3829 -6701.0934 1285.9237 21964.62 -30375.49 301.7594 -1058.1451 1.000375 0.345 + 1315000 -5547.5333 1248.6602 -6796.1935 1434.312 21690.442 -30376.481 299.42789 417.77084 1.0108828 0.3425 + 1320000 -5556.5722 1245.995 -6802.5672 1406.0367 21716.81 -30378.518 298.78879 -44.59583 1.0029205 0.34 + 1325000 -5492.2298 1210.0413 -6702.271 1306.7266 21930.848 -30379.189 290.1671 -360.17536 1.0193503 0.3375 + 1330000 -5529.1732 1201.8706 -6731.0438 1320.2689 21891.221 -30378.853 288.20777 -823.47174 1.0023972 0.335 + 1335000 -5516.1513 1284.0936 -6800.2449 1456.3194 21648.152 -30382.511 307.9248 816.27591 1.0172669 0.3325 + 1340000 -5524.1288 1244.1823 -6768.3111 1437.1822 21726.149 -30377.547 298.3541 659.0745 1.018629 0.33 + 1345000 -5508.6037 1238.7015 -6747.3052 1400.1328 21792.993 -30375.67 297.0398 319.94593 1.0148916 0.3275 + 1350000 -5527.1475 1258.9192 -6786.0666 1356.8947 21794.455 -30379.738 301.88799 -335.38354 1.0081011 0.325 + 1355000 -5538.0487 1250.8881 -6788.9368 1453.457 21678.809 -30378.902 299.96214 769.52717 1.0229997 0.3225 + 1360000 -5476.898 1309.4028 -6786.3009 1427.2196 21697.827 -30381.824 313.99394 803.67442 1.0294179 0.32 + 1365000 -5509.3513 1232.51 -6741.8613 1307.4439 21886.932 -30380.5 295.55508 -1043.7167 1.0072445 0.3175 + 1370000 -5540.0675 1251.3688 -6791.4363 1379.1587 21758.664 -30378.636 300.07741 -156.99758 1.0200723 0.315 + 1375000 -5578.6206 1239.6891 -6818.3097 1350.1415 21780.413 -30376.511 297.27662 -854.76394 1.001508 0.3125 + 1380000 -5463.107 1290.9919 -6754.0989 1444.3422 21730.857 -30378.895 309.579 772.67327 1.0122494 0.31 + 1385000 -5499.3844 1239.3068 -6738.6912 1462.7244 21771.34 -30375.733 297.18495 1283.3351 1.0145078 0.3075 + 1390000 -5636.4741 1214.4694 -6850.9434 1383.584 21682.318 -30380.751 291.22895 -240.05624 1.0213867 0.305 + 1395000 -5537.6011 1252.6604 -6790.2615 1420.8477 21717.595 -30379.56 300.38714 182.01588 1.0064129 0.3025 + 1400000 -5548.9864 1240.0253 -6789.0117 1387.3366 21714.14 -30378.119 297.35725 -395.53785 0.99660636 0.3 + 1405000 -5631.6836 1213.3931 -6845.0766 1394.0426 21727.204 -30380.58 290.97086 -140.02628 1.0191589 0.2975 + 1410000 -5570.2405 1202.1725 -6772.413 1446.4137 21746.694 -30376.362 288.28018 565.38216 0.99852339 0.295 + 1415000 -5513.6068 1260.5019 -6774.1087 1317.293 21880.318 -30377.852 302.26752 -425.54191 1.0260955 0.2925 + 1420000 -5571.7578 1268.6051 -6840.3629 1403.6298 21714.791 -30378.169 304.21066 54.465307 1.0173764 0.29 + 1425000 -5638.9053 1227.4178 -6866.3231 1369.4019 21716.414 -30381.21 294.33398 -489.07663 1.0231539 0.2875 + 1430000 -5484.9163 1258.5341 -6743.4504 1417.5799 21781.385 -30379.25 301.79564 495.28585 1.0123172 0.285 + 1435000 -5513.5471 1284.6685 -6798.2156 1407.8452 21711.668 -30376.834 308.06266 295.19088 1.0173205 0.2825 + 1440000 -5580.8299 1250.3744 -6831.2043 1479.6654 21646.68 -30378.997 299.83895 761.0399 1.0106045 0.28 + 1445000 -5635.3122 1252.8496 -6888.1618 1504.565 21532.223 -30380.129 300.43251 513.57698 1.0076633 0.2775 + 1450000 -5486.7362 1308.6032 -6795.3394 1397.0887 21765.068 -30376.175 313.80218 -99.824485 1.0020521 0.275 + 1455000 -5561.0636 1264.4229 -6825.4865 1399.449 21674.063 -30381.988 303.20778 -415.08661 1.0044893 0.2725 + 1460000 -5602.6724 1237.4201 -6840.0926 1360.0506 21713.944 -30376.542 296.73254 -569.29884 1.0196217 0.27 + 1465000 -5557.9048 1243.4073 -6801.3121 1419.2761 21696.901 -30378.642 298.16825 94.910341 1.0027966 0.2675 + 1470000 -5578.3575 1264.4044 -6842.762 1474.4303 21627.06 -30379.312 303.20335 942.68042 1.0250842 0.265 + 1475000 -5548.4324 1227.8513 -6776.2837 1424.1369 21760.42 -30377.249 294.43793 474.1116 1.0105851 0.2625 + 1480000 -5546.8794 1248.7847 -6795.6641 1341.2731 21811.099 -30381.049 299.45774 -537.89777 1.0220907 0.26 + 1485000 -5469.9253 1303.2112 -6773.1365 1372.6674 21770.592 -30375.734 312.50919 -79.452797 1.0087945 0.2575 + 1490000 -5509.538 1255.0642 -6764.6022 1327.3994 21822.784 -30377.213 300.96356 -808.75259 1.0229172 0.255 + 1495000 -5590.1476 1245.1426 -6835.2902 1380.5927 21711.336 -30377.291 298.58436 -454.99232 1.0007932 0.2525 + 1500000 -5503.2146 1243.1634 -6746.378 1336.568 21869.966 -30374.521 298.10976 -673.77429 1.005844 0.25 + 1505000 -5515.316 1210.9868 -6726.3028 1360.9575 21896.046 -30377.596 290.39383 -117.70175 1.0026611 0.2475 + 1510000 -5466.2688 1291.4793 -6757.7481 1432.113 21744.77 -30378.609 309.69588 559.59585 1.0094784 0.245 + 1515000 -5515.4116 1280.3987 -6795.8103 1454.1004 21725.579 -30379.794 307.03876 976.71744 1.0146273 0.2425 + 1520000 -5566.844 1258.7589 -6825.6029 1328.5982 21795.623 -30380.868 301.84957 -1235.7229 0.99399834 0.24 + 1525000 -5502.8424 1233.1097 -6735.9521 1375.339 21854.059 -30380.66 295.6989 80.767849 1.0122901 0.2375 + 1530000 -5611.7566 1233.4818 -6845.2384 1381.3109 21681.348 -30379.585 295.78811 -497.77083 1.0061235 0.235 + 1535000 -5561.4562 1276.04 -6837.4963 1405.3468 21730.358 -30379.422 305.99356 125.46821 1.0175592 0.2325 + 1540000 -5672.2966 1217.9014 -6890.198 1401.5066 21670.118 -30381.384 292.05195 -243.55974 1.0176228 0.23 + 1545000 -5548.9594 1241.3504 -6790.3098 1322.6254 21791.955 -30377.275 297.67501 -897.01748 1.0094888 0.2275 + 1550000 -5525.9751 1264.4516 -6790.4267 1400.0967 21769.593 -30378.747 303.21467 299.75833 1.0203862 0.225 + 1555000 -5563.0098 1263.0206 -6826.0304 1351.2173 21793.229 -30380.937 302.8715 -578.02252 1.0116829 0.2225 + 1560000 -5485.253 1266.3745 -6751.6275 1368.2641 21821.234 -30376.163 303.67577 -108.52063 1.0065978 0.22 + 1565000 -5536.2445 1243.8651 -6780.1096 1399.694 21773.453 -30381.063 298.27802 155.48476 1.0078165 0.2175 + 1570000 -5571.4171 1246.4551 -6817.8722 1490.8638 21645.287 -30377.619 298.89911 1260.7223 1.0285327 0.215 + 1575000 -5507.1892 1262.1705 -6769.3597 1379.3818 21800.029 -30379.196 302.66766 219.90904 1.0190579 0.2125 + 1580000 -5475.5755 1214.2346 -6689.8101 1360.6564 21878.095 -30381.808 291.17266 162.20674 1.0197691 0.21 + 1585000 -5549.8015 1219.9274 -6769.7289 1406.4392 21759.735 -30380.967 292.53778 283.79685 1.0216043 0.2075 + 1590000 -5469.2857 1251.6933 -6720.9791 1296.634 21940.466 -30375.947 300.15524 -679.87352 1.0116393 0.205 + 1595000 -5551.6405 1237.1996 -6788.8401 1368.9711 21784.35 -30379.239 296.67965 -24.5855 1.0261732 0.2025 + 1600000 -5535.5989 1244.5073 -6780.1062 1458.1539 21706.069 -30377.198 298.43204 755.64074 1.0032465 0.2 + 1605000 -5491.9754 1260.0285 -6752.0039 1392.3315 21805.533 -30375.962 302.15401 223.68891 1.0102672 0.1975 + 1610000 -5502.4313 1282.3536 -6784.7849 1378.1879 21791.234 -30378.942 307.50754 72.219761 1.0226788 0.195 + 1615000 -5434.7653 1258.9645 -6693.7297 1326.2408 21939.287 -30377.143 301.89885 -186.32112 1.0130596 0.1925 + 1620000 -5585.2504 1257.3165 -6842.5669 1345.5246 21744.374 -30378.94 301.50366 -588.30621 1.0274457 0.19 + 1625000 -5528.9293 1241.3212 -6770.2505 1385.8564 21799.552 -30378.87 297.668 -27.78124 1.0087198 0.1875 + 1630000 -5512.0803 1249.7669 -6761.8472 1409.4021 21756.949 -30379.03 299.69328 598.07578 1.0209223 0.185 + 1635000 -5554.4113 1257.7462 -6812.1575 1429.7636 21689.998 -30381.983 301.6067 637.58768 1.0296191 0.1825 + 1640000 -5528.3295 1282.7202 -6811.0497 1425.6194 21721.951 -30382.144 307.59546 201.65193 1.0054558 0.18 + 1645000 -5415.0306 1296.1562 -6711.1868 1359.5794 21836.685 -30373.985 310.8174 -122.49014 1.0025163 0.1775 + 1650000 -5472.535 1279.0671 -6751.6021 1334.4733 21816.614 -30381.683 306.71945 -1149.2916 0.98751404 0.175 + 1655000 -5501.82 1268.5575 -6770.3775 1377.245 21798.047 -30376.936 304.19926 1.7461216 1.0198116 0.1725 + 1660000 -5574.8853 1242.4084 -6817.2937 1402.801 21721.393 -30374.675 297.9287 -368.176 0.98829371 0.17 + 1665000 -5573.5235 1222.8083 -6796.3318 1398.8053 21724.515 -30373.998 293.22863 -34.053733 1.0103876 0.1675 + 1670000 -5465.9501 1315.885 -6781.8351 1342.4906 21789.764 -30380.929 315.54835 -711.01584 1.0050507 0.165 + 1675000 -5587.4957 1204.8035 -6792.2993 1411.9342 21755.712 -30382.222 288.91109 512.51181 1.0285941 0.1625 + 1680000 -5505.6492 1268.1162 -6773.7654 1373.3072 21795.716 -30381.924 304.09344 -65.28487 1.0177793 0.16 + 1685000 -5570.1242 1234.0295 -6804.1537 1333.3621 21779.517 -30377.13 295.91945 -832.3622 1.0233325 0.1575 + 1690000 -5491.4354 1286.9136 -6778.349 1302.901 21867.464 -30379.461 308.60103 -1091.2681 1.0076038 0.155 + 1695000 -5557.2902 1233.4916 -6790.7818 1475.5133 21645.505 -30381.225 295.79047 467.6176 0.98611211 0.1525 + 1700000 -5529.5608 1273.3267 -6802.8875 1385.957 21733.221 -30380.457 305.34291 -234.85874 1.0141094 0.15 + 1705000 -5598.2612 1252.9529 -6851.2142 1368.7434 21746.044 -30380.367 300.45729 -357.4882 1.0229005 0.1475 + 1710000 -5606.2821 1252.1505 -6858.4326 1441.0675 21619.641 -30382.217 300.26485 291.84925 1.0172756 0.145 + 1715000 -5470.884 1270.4438 -6741.3278 1314.3499 21888.17 -30382.544 304.65158 -536.78432 1.0181209 0.1425 + 1720000 -5458.4566 1270.3827 -6728.8393 1414.9094 21772.276 -30379.283 304.63694 383.28792 1.0017567 0.14 + 1725000 -5542.1103 1245.834 -6787.9443 1406.8781 21710.67 -30381.427 298.75018 -196.66899 0.99533902 0.1375 + 1730000 -5546.8538 1225.5416 -6772.3954 1414.5675 21764.375 -30378.796 293.88406 292.19762 1.0062057 0.135 + 1735000 -5533.254 1230.492 -6763.7459 1356.4861 21781.616 -30381.207 295.07116 -279.51002 1.0177131 0.1325 + 1740000 -5516.3203 1269.3866 -6785.7069 1382.6887 21778.338 -30380.038 304.39806 -41.211787 1.0184863 0.13 + 1745000 -5601.6102 1211.6668 -6813.2769 1457.856 21646.431 -30379.268 290.55689 321.57165 1.0044935 0.1275 + 1750000 -5464.07 1262.7335 -6726.8035 1388.3238 21807.797 -30381.51 302.80266 347.94977 1.0199363 0.125 + 1755000 -5609.8158 1220.0117 -6829.8275 1449.8857 21660.858 -30381.398 292.55801 423.13734 1.0091913 0.1225 + 1760000 -5403.3906 1295.0506 -6698.4412 1376.6218 21890.749 -30379.696 310.55227 307.40852 1.0147058 0.12 + 1765000 -5516.242 1249.2409 -6765.4828 1382.647 21758.198 -30377.246 299.56714 167.93234 1.0231885 0.1175 + 1770000 -5582.0367 1264.8589 -6846.8955 1451.9857 21613.33 -30377.524 303.31232 595.73197 1.0185845 0.115 + 1775000 -5537.7575 1229.4186 -6767.1761 1369.0153 21805.496 -30381.159 294.81376 13.517698 1.0260482 0.1125 + 1780000 -5494.5434 1303.1737 -6797.7171 1436.2813 21746.59 -30380.854 312.50019 737.63948 1.019697 0.11 + 1785000 -5506.8968 1245.4135 -6752.3103 1381.9664 21814.635 -30377.185 298.64934 53.072035 1.0092991 0.1075 + 1790000 -5570.6629 1257.0357 -6827.6986 1389.5682 21752.819 -30378.813 301.43633 -254.1322 1.0049075 0.105 + 1795000 -5403.0983 1313.5161 -6716.6144 1299.9894 21921.198 -30378.568 314.98029 -534.90063 1.0142742 0.1025 + 1800000 -5571.5716 1221.6883 -6793.2599 1459.5581 21671.077 -30379.985 292.96006 1199.0646 1.0291174 0.1 + 1805000 -5576.3609 1224.1212 -6800.4821 1353.5425 21807.985 -30380.553 293.54345 -387.59054 1.0179841 0.0975 + 1810000 -5606.1276 1268.2463 -6874.3738 1461.2712 21629.806 -30377.587 304.12462 705.686 1.032963 0.095 + 1815000 -5528.0175 1213.0379 -6741.0554 1404.3111 21772.221 -30376.182 290.88569 360.59394 1.0146256 0.0925 + 1820000 -5519.9749 1246.5714 -6766.5463 1333.2785 21832.117 -30376.792 298.927 -490.34561 1.0222947 0.09 + 1825000 -5445.4758 1293.818 -6739.2938 1310.9077 21908.662 -30376.428 310.25669 -505.90791 1.0175154 0.0875 + 1830000 -5541.7133 1258.3353 -6800.0486 1342.3151 21835.187 -30378.672 301.74797 -431.49631 1.0112617 0.085 + 1835000 -5455.3051 1261.9795 -6717.2846 1352.0659 21888.34 -30378.24 302.62186 -278.9793 1.0005135 0.0825 + 1840000 -5550.1255 1264.0761 -6814.2015 1457.0559 21643.406 -30381.124 303.12461 732.70062 1.0158677 0.08 + 1845000 -5534.0588 1215.4174 -6749.4762 1393.3056 21800.212 -30376.843 291.45629 318.75691 1.0144019 0.0775 + 1850000 -5470.5935 1240.7072 -6711.3007 1398.1611 21825.075 -30377.387 297.52076 246.47011 0.99682722 0.075 + 1855000 -5562.2126 1269.9488 -6832.1614 1405.7155 21717.318 -30382.767 304.5329 158.21904 1.0206821 0.0725 + 1860000 -5593.1478 1212.5719 -6805.7197 1431.8435 21688.522 -30377.259 290.77394 490.69887 1.0155469 0.07 + 1865000 -5547.7314 1230.6551 -6778.3865 1415.4128 21748.348 -30378.118 295.11029 184.13907 0.99833163 0.0675 + 1870000 -5530.8677 1219.9933 -6750.861 1379.1571 21831.652 -30375.929 292.55359 34.766398 1.0094217 0.065 + 1875000 -5568.1541 1254.3638 -6822.5179 1355.1579 21740.634 -30379.983 300.7956 -705.50446 1.0143892 0.0625 + 1880000 -5653.4584 1235.3915 -6888.8499 1430.7173 21633.442 -30379.735 296.24606 102.70794 1.0154835 0.06 + 1885000 -5465.2275 1218.1694 -6683.3969 1239.8122 21974.299 -30378.879 292.11621 -1526.9501 1.00297 0.0575 + 1890000 -5596.8191 1179.0514 -6775.8705 1375.6822 21763.894 -30376.484 282.73574 -135.80694 1.0180663 0.055 + 1895000 -5465.5228 1216.5423 -6682.0651 1399.7408 21824.803 -30381.567 291.72604 189.86203 0.9918745 0.0525 + 1900000 -5525.4584 1268.983 -6794.4415 1386.5598 21763.346 -30379.48 304.30129 -147.74084 1.0101088 0.05 + 1905000 -5629.8529 1254.2275 -6884.0804 1455.3465 21571.828 -30380.574 300.76293 536.16177 1.0309795 0.0475 + 1910000 -5498.7043 1240.937 -6739.6413 1343.971 21867.501 -30378.531 297.57587 -307.05181 1.00532 0.045 + 1915000 -5489.6666 1266.2052 -6755.8718 1356.677 21815.921 -30373.873 303.63517 41.753289 1.0289101 0.0425 + 1920000 -5534.4011 1264.7423 -6799.1433 1414.4972 21675.646 -30381.358 303.28436 307.25885 1.030805 0.04 + 1925000 -5504.0398 1249.1105 -6753.1503 1407.4242 21780.765 -30377.439 299.53588 416.81022 1.0112989 0.0375 + 1930000 -5486.5931 1291.1596 -6777.7527 1380.6453 21793.976 -30378.396 309.61922 -167.68264 1.0083434 0.035 + 1935000 -5482.2055 1242.0698 -6724.2753 1296.2668 21881.772 -30378.102 297.84751 -1240.512 1.0083689 0.0325 + 1940000 -5413.7616 1287.0265 -6700.788 1300.618 21943.573 -30379.549 308.6281 -822.10018 0.9947525 0.03 + 1945000 -5618.7196 1214.3524 -6833.072 1410.8621 21689.242 -30381.323 291.2009 177.78285 1.0261522 0.0275 + 1950000 -5559.1869 1277.5888 -6836.7757 1384.1924 21715.907 -30380.281 306.36495 -269.07872 1.01053 0.025 + 1955000 -5556.8805 1290.1761 -6847.0566 1420.3306 21702.458 -30379.354 309.38338 63.181765 1.0037638 0.0225 + 1960000 -5536.3473 1288.352 -6824.6993 1333.2341 21766.427 -30380.118 308.94596 -956.94729 1.0098248 0.02 + 1965000 -5560.7629 1271.8239 -6832.5868 1394.6008 21745.026 -30375.628 304.98253 27.022116 1.014722 0.0175 + 1970000 -5571.8348 1246.9662 -6818.801 1368.7901 21728.521 -30376.246 299.02167 -642.85981 1.0038267 0.015 + 1975000 -5448.1873 1277.5391 -6725.7263 1376.1799 21855.417 -30379.445 306.35302 158.55616 1.010898 0.0125 + 1980000 -5569.4362 1227.1456 -6796.5818 1384.1163 21747.316 -30379.52 294.26871 -224.4952 1.0128917 0.01 + 1985000 -5490.1779 1256.3196 -6746.4975 1376.0987 21800.479 -30378.853 301.26461 -23.927186 1.0117616 0.0075 + 1990000 -5568.4743 1216.0821 -6784.5564 1365.6444 21804.572 -30380.078 291.61568 -40.531986 1.024747 0.005 + 1995000 -5623.9072 1244.7555 -6868.6627 1444.7105 21613.58 -30377.599 298.49155 401.54781 1.0221617 0.0025 + 2000000 -5564.5636 1253.8102 -6818.3738 1394.5734 21742.16 -30376.703 300.66286 -89.748646 1.0114229 0 +Loop time of 13852.8 on 12 procs for 2000000 steps with 1800 atoms + +Performance: 12.474 ns/day, 1.924 hours/ns, 144.375 timesteps/s +95.5% CPU use with 12 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 7431.1 | 8143.6 | 8773.3 | 379.5 | 58.79 +Bond | 24.35 | 26.055 | 27.801 | 23.2 | 0.19 +Kspace | 2212.7 | 2764.8 | 3391.2 | 573.6 | 19.96 +Neigh | 568.42 | 569.87 | 570.75 | 2.7 | 4.11 +Comm | 806.14 | 866.5 | 930.63 | 180.1 | 6.26 +Output | 0.032719 | 0.034078 | 0.041882 | 1.3 | 0.00 +Modify | 964.98 | 1144.9 | 1300.5 | 428.3 | 8.26 +Other | | 337 | | | 2.43 + +Nlocal: 150.000 ave 166 max 139 min +Histogram: 3 0 2 0 2 3 0 0 1 1 +Nghost: 6125.92 ave 6158 max 6091 min +Histogram: 1 1 0 2 1 2 2 1 1 1 +Neighs: 87012.7 ave 102741 max 79216 min +Histogram: 2 1 2 3 3 0 0 0 0 1 + +Total # of neighbors = 1044152 +Ave neighs/atom = 580.08444 +Ave special neighs/atom = 2.0000000 +Neighbor list builds = 89145 +Dangerous builds = 0 +Total wall time: 4:01:57 diff --git a/examples/USER/fep/SPCEhyd/fep10/fep10-q.fep b/examples/USER/fep/SPCEhyd/fep10/fep10-q.fep new file mode 100644 index 0000000000..fd49769ec2 --- /dev/null +++ b/examples/USER/fep/SPCEhyd/fep10/fep10-q.fep @@ -0,0 +1,22 @@ +# Time-averaged data for fix FEP +# TimeStep c_FEP[1] c_FEP[2] c_FEP[3] +100000 1.45025 1741.09 17759.6 +200000 1.34275 2079.09 17751.2 +300000 1.11463 3077.67 17744.5 +400000 1.05072 3418.61 17752.1 +500000 0.861524 4545.87 17736.9 +600000 0.752417 5411.55 17755.8 +700000 0.677848 6201.52 17715.7 +800000 0.594413 7014.15 17718.2 +900000 0.482601 8441.94 17738.8 +1000000 0.44748 8942.75 17761.6 +1100000 0.374076 9984.65 17731.4 +1200000 0.310756 11050 17774.3 +1300000 0.246639 12186.7 17783.6 +1400000 0.201753 13117.8 17825.2 +1500000 0.166052 13843.8 17784.7 +1600000 0.136584 14486.5 17742.2 +1700000 0.102778 15331.6 17778.7 +1800000 0.0716746 16103 17743.1 +1900000 0.0513218 16698.6 17801.1 +2000000 0.0250996 17424.1 17780.5 diff --git a/examples/USER/fep/SPCEhyd/fep10/fep10-q.out b/examples/USER/fep/SPCEhyd/fep10/fep10-q.out new file mode 100644 index 0000000000..36fc89e567 --- /dev/null +++ b/examples/USER/fep/SPCEhyd/fep10/fep10-q.out @@ -0,0 +1,559 @@ +LAMMPS (24 Dec 2020) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (src/lammps/src/comm.cpp:94) + using 1 OpenMP thread(s) per MPI task +Reading data file ... + orthogonal box = (0.0000000 0.0000000 0.0000000) to (29.204526 29.204526 29.204526) + 2 by 2 by 3 MPI processor grid + reading atoms ... + 1800 atoms + scanning bonds ... + 1 = max bonds/atom + scanning angles ... + 1 = max angles/atom + reading bonds ... + 1200 bonds + reading angles ... + 600 angles +Finding 1-2 1-3 1-4 neighbors ... + special bond factors lj: 0 0 0.5 + special bond factors coul: 0 0 0.5 + 2 = max # of 1-2 neighbors + 1 = max # of 1-3 neighbors + 1 = max # of 1-4 neighbors + 2 = max # of special neighbors + special bonds CPU = 0.002 seconds + read_data CPU = 0.020 seconds +Finding SHAKE clusters ... + 0 = # of size 2 clusters + 600 = # of size 3 clusters + 0 = # of size 4 clusters + 0 = # of frozen angles + find clusters CPU = 0.001 seconds +PPPM initialization ... + using 12-bit tables for long-range coulomb (src/lammps/src/kspace.cpp:339) + G vector (1/distance) = 0.2506685 + grid = 20 20 20 + stencil order = 5 + estimated absolute RMS force accuracy = 0.0030770287 + estimated relative force accuracy = 9.2663804e-06 + using double precision FFTW3 + 3d grid and FFT values/proc = 3757 800 +Neighbor list info ... + update every 1 steps, delay 10 steps, check yes + max neighbors/atom: 2000, page size: 100000 + master list distance cutoff = 14 + ghost atom cutoff = 14 + binsize = 7, bins = 5 5 5 + 1 neighbor lists, perpetual/occasional/extra = 1 0 0 + (1) pair lj/cut/coul/long, perpetual + attributes: half, newton on + pair build: half/bin/newton + stencil: half/bin/3d/newton + bin: standard +Setting up Verlet run ... + Unit style : real + Current step : 0 + Time step : 1 +Per MPI rank memory allocation (min/avg/max) = 8.214 | 8.226 | 8.244 Mbytes +Step CPU TotEng KinEng PotEng E_vdwl E_coul E_long Temp Press Volume Density + 0 0 1343.8082 1251.046 92.762256 53.823145 30098.377 -30059.438 300 10285.006 24908.667 0.72058313 + 5000 27.484029 -5481.5313 1291.2507 -6772.7821 1417.9265 21710.404 -30351.578 309.64107 573.94807 17792.658 1.0087737 + 10000 51.222479 -5557.9174 1247.8414 -6805.7588 1463.5652 21652.169 -30349.483 299.23155 1155.9001 17391.932 1.0320167 + 15000 81.534825 -5547.6429 1255.8047 -6803.4476 1434.9211 21679.801 -30351.564 301.14114 414.89266 17774.479 1.0098054 + 20000 108.19472 -5520.4365 1226.2567 -6746.6933 1363.6958 21763.102 -30350.115 294.05555 -159.5117 17640.377 1.017482 + 25000 139.47439 -5523.2547 1214.4617 -6737.7163 1403.2241 21727.256 -30347.421 291.2271 682.49951 17446.971 1.0287611 + 30000 173.52324 -5633.4887 1209.4695 -6842.9582 1384.9611 21696.903 -30350.921 290.02999 -439.40705 17870.41 1.0043846 + 35000 207.46148 -5609.1629 1202.7623 -6811.9252 1459.4041 21642.871 -30348.683 288.4216 870.04629 17620.845 1.0186098 + 40000 240.785 -5613.2727 1218.043 -6831.3157 1458.2338 21629.898 -30351.078 292.08591 578.74278 17728.011 1.0124523 + 45000 274.86574 -5567.9001 1234.3625 -6802.2626 1499.8464 21612.157 -30349.505 295.9993 1143.6708 17911.841 1.0020614 + 50000 308.33717 -5534.5804 1253.1529 -6787.7333 1383.4124 21740.111 -30349.237 300.50525 -236.43099 17808.85 1.0078565 + 55000 340.35767 -5558.5567 1237.2613 -6795.818 1472.1262 21640.69 -30349.9 296.69444 1169.6084 17518.859 1.0245396 + 60000 371.42788 -5620.3085 1217.3007 -6837.6091 1335.6617 21764.982 -30350.897 291.9079 -730.47206 17652.428 1.0167873 + 65000 400.4258 -5523.4332 1219.0927 -6742.5258 1377.9207 21790.846 -30351.438 292.33762 -67.377992 18008.535 0.99668103 + 70000 433.12056 -5549.9652 1278.9899 -6828.9551 1405.3901 21680.053 -30344.054 306.70094 -24.208971 17740.923 1.0117154 + 75000 465.43309 -5468.4134 1328.8758 -6797.2892 1404.3997 21721.55 -30346.155 318.66353 422.45424 17600.531 1.0197854 + 80000 498.33654 -5561.5506 1262.2191 -6823.7698 1375.8895 21728.477 -30350.941 302.67932 -480.08789 17936.63 1.0006765 + 85000 531.18843 -5561.7498 1243.7135 -6805.4633 1411.6336 21707.72 -30349.917 298.24169 31.756571 17844.951 1.0058176 + 90000 565.37172 -5548.2883 1305.5336 -6853.8219 1376.838 21698.95 -30345.634 313.06609 -580.39211 18032.442 0.99535964 + 95000 599.1335 -5491.6607 1249.0766 -6740.7373 1327.508 21805.341 -30346.173 299.52775 -747.34141 17928.72 1.001118 + 100000 632.97763 -5504.9781 1270.9776 -6775.9557 1348.4638 21759.337 -30349.51 304.7796 -112.23793 17469.352 1.0274431 +Loop time of 632.978 on 12 procs for 100000 steps with 1800 atoms + +Performance: 13.650 ns/day, 1.758 hours/ns, 157.983 timesteps/s +95.0% CPU use with 12 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 307.42 | 344.2 | 378.91 | 110.2 | 54.38 +Bond | 1.1476 | 1.2339 | 1.3345 | 6.3 | 0.19 +Kspace | 109.33 | 144.47 | 180.71 | 170.1 | 22.82 +Neigh | 25.619 | 25.671 | 25.708 | 0.6 | 4.06 +Comm | 41.234 | 45.135 | 50.464 | 52.8 | 7.13 +Output | 0.0012664 | 0.001324 | 0.0018066 | 0.4 | 0.00 +Modify | 46.059 | 59.064 | 69.016 | 114.7 | 9.33 +Other | | 13.2 | | | 2.09 + +Nlocal: 150.000 ave 160 max 138 min +Histogram: 2 0 0 2 1 1 3 0 0 3 +Nghost: 6169.08 ave 6215 max 6126 min +Histogram: 1 1 2 2 0 1 3 0 1 1 +Neighs: 88745.0 ave 98807 max 82382 min +Histogram: 3 0 1 3 0 3 1 0 0 1 + +Total # of neighbors = 1064940 +Ave neighs/atom = 591.63333 +Ave special neighs/atom = 2.0000000 +Neighbor list builds = 4480 +Dangerous builds = 8 +PPPM initialization ... + using 12-bit tables for long-range coulomb (src/lammps/src/kspace.cpp:339) + G vector (1/distance) = 0.25148052 + grid = 18 18 18 + stencil order = 5 + estimated absolute RMS force accuracy = 0.0034663752 + estimated relative force accuracy = 1.0438886e-05 + using double precision FFTW3 + 3d grid and FFT values/proc = 3328 648 +FEP settings ... + temperature = 300.000000 + tail no + 1-1 charge + 2-2 charge +Setting up Verlet run ... + Unit style : real + Current step : 0 + Time step : 1 +Per MPI rank memory allocation (min/avg/max) = 8.210 | 8.549 | 8.629 Mbytes +Step TotEng KinEng PotEng E_vdwl E_coul E_long Temp Press Density v_lambda v_qO v_qH + 0 -5504.9889 1270.9776 -6775.9665 1348.4638 21857.535 -30447.719 304.7796 -83.574867 1.0274431 1 -0.8476 0.4238 + 5000 -5524.9498 1270.0815 -6795.0313 1351.1305 21827.149 -30444.933 304.5647 -593.70775 1.005159 0.9975 -0.845481 0.4227405 + 10000 -5540.3394 1230.4582 -6770.7976 1419.9827 21833.217 -30447.512 295.06306 390.92433 1.0060179 0.995 -0.843362 0.421681 + 15000 -5552.7887 1238.4753 -6791.264 1335.4158 21841.316 -30448.525 296.98555 -977.57176 1.0018216 0.9925 -0.841243 0.4206215 + 20000 -5571.4362 1242.7492 -6814.1854 1430.7811 21758.542 -30446.37 298.01044 389.91739 1.0176182 0.99 -0.839124 0.419562 + 25000 -5460.6807 1271.7643 -6732.445 1445.8579 21824.91 -30444.435 304.96824 662.28347 0.99362968 0.9875 -0.837005 0.4185025 + 30000 -5454.6572 1289.2653 -6743.9226 1383.6837 21875.483 -30446.254 309.16498 76.696731 1.0117008 0.985 -0.834886 0.417443 + 35000 -5460.3689 1262.5044 -6722.8733 1424.7799 21854.486 -30444.887 302.74772 469.28759 0.99223648 0.9825 -0.832767 0.4163835 + 40000 -5617.7692 1259.5855 -6877.3547 1466.8238 21653.475 -30448.558 302.04777 518.56945 1.0149626 0.98 -0.830648 0.415324 + 45000 -5498.7892 1260.7599 -6759.5491 1354.8241 21911.823 -30445.433 302.32939 36.724958 1.0199193 0.9775 -0.828529 0.4142645 + 50000 -5556.2521 1276.3605 -6832.6126 1408.5453 21757.203 -30447.43 306.0704 38.976819 1.012915 0.975 -0.82641 0.413205 + 55000 -5513.8511 1239.4864 -6753.3375 1378.8961 21910.688 -30446.837 297.22802 -155.09229 0.99684598 0.9725 -0.824291 0.4121455 + 60000 -5482.86 1287.2484 -6770.1084 1383.4644 21876.85 -30446.035 308.68131 -23.984269 1.002056 0.97 -0.822172 0.411086 + 65000 -5556.4674 1243.4649 -6799.9323 1374.1364 21841.845 -30449.652 298.18207 13.80546 1.0185986 0.9675 -0.820053 0.4100265 + 70000 -5509.3689 1255.8787 -6765.2476 1342.6439 21905.378 -30448.437 301.15889 -636.57169 1.0013923 0.965 -0.817934 0.408967 + 75000 -5599.5463 1235.6343 -6835.1807 1391.5344 21779.525 -30444.409 296.3043 -272.25195 1.0096277 0.9625 -0.815815 0.4079075 + 80000 -5505.7102 1228.1399 -6733.8502 1402.1931 21862.826 -30444.663 294.50714 287.56695 1.007265 0.96 -0.813696 0.406848 + 85000 -5615.9836 1228.8806 -6844.8643 1412.2136 21728.301 -30447.357 294.68477 88.808898 1.0201101 0.9575 -0.811577 0.4057885 + 90000 -5445.8525 1286.0493 -6731.9017 1365.0964 21944.508 -30446.123 308.39376 -159.5813 0.99818678 0.955 -0.809458 0.404729 + 95000 -5555.6599 1255.1923 -6810.8522 1487.4112 21697.426 -30446.382 300.99428 1056.7026 1.0047794 0.9525 -0.807339 0.4036695 + 100000 -5423.2205 1299.4514 -6722.6719 1340.3931 21969.252 -30445.58 311.60759 -104.22862 1.0205672 0.95 -0.80522 0.40261 + 105000 -5510.0668 1219.7044 -6729.7712 1422.1714 21857.811 -30444.935 292.48431 965.21304 1.0267571 0.9475 -0.803101 0.4015505 + 110000 -5487.6062 1264.3775 -6751.9838 1383.7805 21872.105 -30438.176 303.19689 5.0812841 1.0038721 0.945 -0.800982 0.400491 + 115000 -5584.4825 1223.5314 -6808.0139 1305.413 21904.363 -30442.927 293.40202 -578.05444 1.0328242 0.9425 -0.798863 0.3994315 + 120000 -5523.0074 1287.4784 -6810.4858 1458.6493 21678.215 -30441.67 308.73646 671.54744 1.0055922 0.94 -0.796744 0.398372 + 125000 -5523.9631 1261.6061 -6785.5692 1451.4065 21774.541 -30440.463 302.5323 943.71106 1.023251 0.9375 -0.794625 0.3973125 + 130000 -5480.6587 1228.3138 -6708.9725 1318.5357 22013.43 -30444.357 294.54883 -484.16808 1.0024483 0.935 -0.792506 0.396253 + 135000 -5515.4922 1236.63 -6752.1222 1405.2286 21838.239 -30444.636 296.54306 -33.044871 0.99099073 0.9325 -0.790387 0.3951935 + 140000 -5636.9585 1190.5857 -6827.5442 1401.0211 21837.041 -30443.664 285.50167 -131.26918 1.0073498 0.93 -0.788268 0.394134 + 145000 -5520.4299 1242.8162 -6763.246 1425.5354 21801.806 -30441.82 298.02649 375.70839 1.0055963 0.9275 -0.786149 0.3930745 + 150000 -5466.2108 1221.5587 -6687.7695 1315.243 22002.492 -30445.561 292.92898 -458.87734 1.0111682 0.925 -0.78403 0.392015 + 155000 -5461.822 1239.1272 -6700.9492 1371.2953 21931.804 -30440.608 297.14189 404.63471 1.02125 0.9225 -0.781911 0.3909555 + 160000 -5581.5531 1235.5332 -6817.0862 1387.24 21799.458 -30439.508 296.28004 -313.78406 1.005984 0.92 -0.779792 0.389896 + 165000 -5518.8078 1273.6502 -6792.4581 1375.9673 21862.55 -30442.152 305.42049 -35.844433 1.0122448 0.9175 -0.777673 0.3888365 + 170000 -5515.2831 1285.6763 -6800.9594 1404.6207 21813.634 -30442.455 308.30433 -58.986086 1.010512 0.915 -0.775554 0.387777 + 175000 -5547.8632 1237.0346 -6784.8979 1371.988 21866.075 -30441.096 296.64009 8.7538748 1.0204871 0.9125 -0.773435 0.3867175 + 180000 -5550.9154 1284.8683 -6835.7837 1386.3915 21803.513 -30444.858 308.11056 107.40069 1.0274352 0.91 -0.771316 0.385658 + 185000 -5536.9233 1270.2304 -6807.1538 1398.1504 21809.601 -30443.002 304.60042 -174.16652 0.99755738 0.9075 -0.769197 0.3845985 + 190000 -5518.9253 1280.5079 -6799.4332 1373.9909 21862.478 -30442.723 307.06496 9.6282943 1.0156746 0.905 -0.767078 0.383539 + 195000 -5574.7502 1214.359 -6789.1092 1411.3746 21807.743 -30442.797 291.20249 459.43071 1.0173851 0.9025 -0.764959 0.3824795 + 200000 -5586.1746 1230.4227 -6816.5973 1532.8087 21640.232 -30440.345 295.05456 1663.9344 1.0127589 0.9 -0.76284 0.38142 + 205000 -5530.762 1262.3017 -6793.0637 1421.2001 21765.141 -30437.136 302.69912 362.504 1.0112973 0.8975 -0.760721 0.3803605 + 210000 -5482.179 1264.6727 -6746.8517 1332.5039 21878.405 -30436.424 303.26767 -518.59319 1.0144674 0.895 -0.758602 0.379301 + 215000 -5548.4664 1254.2078 -6802.6742 1439.71 21751.539 -30437.604 300.7582 936.14316 1.0295052 0.8925 -0.756483 0.3782415 + 220000 -5566.4852 1224.3179 -6790.8031 1366.173 21833.016 -30437.139 293.59062 -368.10245 1.0080289 0.89 -0.754364 0.377182 + 225000 -5607.3792 1195.1042 -6802.4834 1434.8209 21716.922 -30436.88 286.58521 313.99343 1.0126994 0.8875 -0.752245 0.3761225 + 230000 -5685.3593 1214.5576 -6899.9169 1452.8971 21628.537 -30433.937 291.2501 -34.04353 1.0037164 0.885 -0.750126 0.375063 + 235000 -5458.1208 1277.1379 -6735.2588 1388.7878 21861.855 -30438.247 306.25684 26.373018 1.0027032 0.8825 -0.748007 0.3740035 + 240000 -5471.8252 1281.8041 -6753.6293 1442.5812 21792.96 -30437.921 307.37578 706.48275 1.0008045 0.88 -0.745888 0.372944 + 245000 -5459.3621 1258.1055 -6717.4676 1394.4965 21879.415 -30435.773 301.69286 446.21806 1.0155238 0.8775 -0.743769 0.3718845 + 250000 -5614.6519 1198.1053 -6812.7572 1397.7454 21767.081 -30436.857 287.30486 146.62072 1.0273533 0.875 -0.74165 0.370825 + 255000 -5573.1837 1250.4489 -6823.6325 1354.8074 21829.39 -30434.368 299.85681 -608.07208 1.0009113 0.8725 -0.739531 0.3697655 + 260000 -5630.451 1202.8398 -6833.2907 1380.657 21778.541 -30440.821 288.44019 -195.24451 1.019413 0.87 -0.737412 0.368706 + 265000 -5533.5948 1273.5591 -6807.1539 1419.8611 21794.887 -30438.706 305.39863 339.13963 1.0138164 0.8675 -0.735293 0.3676465 + 270000 -5640.803 1204.5526 -6845.3556 1495.5078 21652.609 -30438.707 288.85092 1463.7932 1.0356466 0.865 -0.733174 0.366587 + 275000 -5514.7021 1283.4057 -6798.1078 1340.6427 21864.301 -30436.815 307.75984 -449.32236 1.0128657 0.8625 -0.731055 0.3655275 + 280000 -5610.2442 1259.5217 -6869.7659 1411.1009 21686.723 -30440.226 302.03248 -123.59673 1.0128803 0.86 -0.728936 0.364468 + 285000 -5550.4897 1281.3066 -6831.7963 1373.2421 21810.977 -30438.399 307.25647 -248.99043 1.0185983 0.8575 -0.726817 0.3634085 + 290000 -5553.4882 1240.2591 -6793.7473 1391.9459 21802.734 -30437.51 297.4133 -158.86961 1.0031364 0.855 -0.724698 0.362349 + 295000 -5524.2791 1255.0193 -6779.2984 1449.4799 21776.57 -30434.687 300.95281 952.9175 1.0131377 0.8525 -0.722579 0.3612895 + 300000 -5476.0212 1284.8796 -6760.9008 1324.2729 21929.73 -30436.686 308.11327 -465.3353 1.0043923 0.85 -0.72046 0.36023 + 305000 -5438.0362 1298.3446 -6736.3809 1407.4391 21826.292 -30430.432 311.34219 517.43655 1.0077139 0.8475 -0.718341 0.3591705 + 310000 -5535.7453 1264.3612 -6800.1065 1343.2269 21832.58 -30432.621 303.19298 -745.84969 1.0074616 0.845 -0.716222 0.358111 + 315000 -5502.0889 1225.373 -6727.4619 1408.6878 21868.726 -30430.42 293.84363 391.97607 1.0057154 0.8425 -0.714103 0.3570515 + 320000 -5551.1885 1247.6582 -6798.8467 1441.5698 21745.518 -30434.212 299.18761 602.66464 1.0106275 0.84 -0.711984 0.355992 + 325000 -5472.7566 1268.0116 -6740.7682 1396.6033 21874.12 -30434.532 304.06836 68.811265 0.9970358 0.8375 -0.709865 0.3549325 + 330000 -5572.2981 1246.5687 -6818.8668 1407.6183 21773.909 -30431.719 298.92634 224.12439 1.0246504 0.835 -0.707746 0.353873 + 335000 -5499.3422 1205.6434 -6704.9856 1285.2624 21998.798 -30434.913 289.11249 -817.85014 1.0147781 0.8325 -0.705627 0.3528135 + 340000 -5562.2169 1219.1939 -6781.4108 1417.0604 21795.894 -30433.607 292.36189 174.41821 1.0058188 0.83 -0.703508 0.351754 + 345000 -5520.3776 1219.9601 -6740.3377 1367.4757 21877.315 -30431.491 292.54563 -222.24612 1.0043509 0.8275 -0.701389 0.3506945 + 350000 -5552.9644 1291.4697 -6844.4341 1376.2399 21787.524 -30434.036 309.69358 -160.81114 1.0193989 0.825 -0.69927 0.349635 + 355000 -5592.6369 1199.1021 -6791.739 1447.0154 21758.057 -30432.531 287.54389 514.62059 1.0124065 0.8225 -0.697151 0.3485755 + 360000 -5529.6905 1251.9083 -6781.5988 1413.2964 21787.146 -30433.685 300.20679 176.70328 1.0073792 0.82 -0.695032 0.347516 + 365000 -5555.2054 1274.6375 -6829.8429 1394.8131 21789.095 -30432.379 305.65724 -24.957537 1.0152213 0.8175 -0.692913 0.3464565 + 370000 -5549.5587 1233.5667 -6783.1253 1413.7135 21800.689 -30434.305 295.80847 536.16714 1.0185709 0.815 -0.690794 0.345397 + 375000 -5600.3525 1246.9961 -6847.3486 1429.7166 21731.282 -30431.632 299.02885 153.53182 1.0080114 0.8125 -0.688675 0.3443375 + 380000 -5546.5526 1248.9012 -6795.4538 1492.7679 21715.426 -30433.665 299.48569 1426.4389 1.0177045 0.81 -0.686556 0.343278 + 385000 -5526.007 1233.736 -6759.7429 1424.1481 21831.876 -30433.42 295.84907 794.1626 1.0185785 0.8075 -0.684437 0.3422185 + 390000 -5596.7029 1243.227 -6839.9299 1421.1174 21693.364 -30433.154 298.12502 108.87841 1.0093018 0.805 -0.682318 0.341159 + 395000 -5546.0663 1265.2707 -6811.337 1447.5582 21710.518 -30434.09 303.41109 435.10835 1.0032239 0.8025 -0.680199 0.3400995 + 400000 -5512.3512 1228.2014 -6740.5526 1323.056 21887.642 -30435.079 294.52189 -597.96971 1.0128567 0.8 -0.67808 0.33904 + 405000 -5452.0555 1256.5329 -6708.5884 1343.4033 21932.615 -30422.632 301.31577 -151.04994 1.0109662 0.7975 -0.675961 0.3379805 + 410000 -5564.1544 1254.1139 -6818.2683 1392.8911 21749.311 -30425.522 300.73568 -588.71236 0.99221624 0.795 -0.673842 0.336921 + 415000 -5492.9731 1270.5302 -6763.5033 1406.1534 21801.397 -30431.784 304.6723 283.61097 1.0125944 0.7925 -0.671723 0.3358615 + 420000 -5595.3267 1203.6829 -6799.0097 1342.6073 21860.776 -30429.796 288.64237 -326.4097 1.0197083 0.79 -0.669604 0.334802 + 425000 -5526.916 1270.6811 -6797.5971 1386.3023 21838.644 -30429.504 304.70849 59.964397 1.0107947 0.7875 -0.667485 0.3337425 + 430000 -5463.4519 1253.2065 -6716.6584 1368.6563 21900.489 -30429.468 300.51809 163.25845 1.0046663 0.785 -0.665366 0.332683 + 435000 -5587.8678 1212.7905 -6800.6583 1357.107 21806.156 -30427.795 290.82636 -669.59026 1.0055054 0.7825 -0.663247 0.3316235 + 440000 -5579.1616 1230.2594 -6809.421 1367.4502 21836.582 -30425.594 295.01539 -322.05693 1.0101379 0.78 -0.661128 0.330564 + 445000 -5532.555 1201.8734 -6734.4284 1386.7876 21878.725 -30427.156 288.20845 -45.792864 1.0052685 0.7775 -0.659009 0.3295045 + 450000 -5569.8426 1278.6957 -6848.5383 1414.4583 21749.442 -30430.096 306.63038 236.64985 1.0219043 0.775 -0.65689 0.328445 + 455000 -5449.1927 1264.9233 -6714.116 1326.8274 21933.446 -30429.11 303.32777 -442.03731 0.99413577 0.7725 -0.654771 0.3273855 + 460000 -5554.4541 1270.3142 -6824.7683 1333.1464 21840.149 -30429.094 304.6205 -747.1109 1.0160003 0.77 -0.652652 0.326326 + 465000 -5528.8036 1284.1598 -6812.9634 1380.0206 21775.39 -30428.738 307.94068 -20.995822 1.0245304 0.7675 -0.650533 0.3252665 + 470000 -5512.8124 1236.1415 -6748.9539 1406.7843 21790.482 -30429.605 296.42591 302.60013 1.0139043 0.765 -0.648414 0.324207 + 475000 -5498.2571 1273.8167 -6772.0739 1435.6038 21790.464 -30429.5 305.46041 483.68473 1.0001103 0.7625 -0.646295 0.3231475 + 480000 -5559.1482 1225.6395 -6784.7878 1384.7454 21844.676 -30428.007 293.90755 30.99406 1.0121504 0.76 -0.644176 0.322088 + 485000 -5496.7804 1282.5658 -6779.3462 1415.6133 21800.486 -30429.421 307.55843 530.11667 1.0187969 0.7575 -0.642057 0.3210285 + 490000 -5537.517 1233.6775 -6771.1945 1371.616 21849.424 -30427.207 295.83505 -358.41182 0.99575692 0.755 -0.639938 0.319969 + 495000 -5554.5243 1212.6181 -6767.1424 1404.0842 21816.065 -30425.237 290.78503 209.62449 1.0112618 0.7525 -0.637819 0.3189095 + 500000 -5622.7664 1199.6165 -6822.3829 1327.3169 21871.075 -30428.955 287.66724 -1311.6743 0.99857938 0.75 -0.6357 0.31785 + 505000 -5480.3266 1256.8007 -6737.1273 1299.6547 21957.857 -30425.971 301.37997 -899.16025 1.0089969 0.7475 -0.633581 0.3167905 + 510000 -5498.9495 1225.5254 -6724.4749 1348.4413 21919.082 -30425.115 293.88018 -394.00866 1.006416 0.745 -0.631462 0.315731 + 515000 -5551.2688 1235.3365 -6786.6053 1382.192 21812.545 -30425.295 296.23288 -234.82847 1.000347 0.7425 -0.629343 0.3146715 + 520000 -5580.0926 1192.4884 -6772.581 1455.062 21751.768 -30422.995 285.95793 899.21376 1.0184475 0.74 -0.627224 0.313612 + 525000 -5536.1025 1254.7959 -6790.8984 1402.6731 21772.842 -30423.645 300.89923 -179.78368 1.0012739 0.7375 -0.625105 0.3125525 + 530000 -5475.5649 1274.227 -6749.7919 1422.191 21806.648 -30423.468 305.5588 412.66908 1.0021064 0.735 -0.622986 0.311493 + 535000 -5535.7717 1220.4498 -6756.2215 1277.0625 21937.566 -30422.661 292.66305 -1373.8334 1.0012496 0.7325 -0.620867 0.3104335 + 540000 -5559.6489 1229.0591 -6788.708 1373.477 21802.469 -30424.781 294.72757 -356.60453 1.0077789 0.73 -0.618748 0.309374 + 545000 -5550.6131 1217.3371 -6767.9502 1356.1731 21855.265 -30426.343 291.91664 -410.40203 1.0164552 0.7275 -0.616629 0.3083145 + 550000 -5552.1925 1225.9662 -6778.1588 1407.9503 21859.108 -30426.047 293.98589 371.95292 1.0041485 0.725 -0.61451 0.307255 + 555000 -5474.3141 1269.9785 -6744.2926 1380.6756 21869.918 -30423.625 304.54001 479.97718 1.0348696 0.7225 -0.612391 0.3061955 + 560000 -5529.7258 1228.0526 -6757.7784 1371.1262 21873.308 -30423.612 294.4862 10.021913 1.0178349 0.72 -0.610272 0.305136 + 565000 -5578.7184 1238.3238 -6817.0421 1364.6801 21831.12 -30424.526 296.94922 -210.96144 1.0145779 0.7175 -0.608153 0.3040765 + 570000 -5590.3739 1178.8219 -6769.1958 1342.169 21850.785 -30426.607 282.68072 -726.65935 1.0040162 0.715 -0.606034 0.303017 + 575000 -5538.2977 1301.251 -6839.5488 1351.7057 21811.514 -30423.38 312.03914 -697.06828 1.0043954 0.7125 -0.603915 0.3019575 + 580000 -5489.5408 1264.601 -6754.1418 1361.3606 21879.773 -30427.021 303.25049 0.8028479 1.0182595 0.71 -0.601796 0.300898 + 585000 -5564.4144 1237.0831 -6801.4975 1372.1667 21823.237 -30425.788 296.6517 -219.05406 0.99937042 0.7075 -0.599677 0.2998385 + 590000 -5642.7568 1231.4142 -6874.171 1428.0404 21658.491 -30425.167 295.29231 186.19832 1.0221448 0.705 -0.597558 0.298779 + 595000 -5623.2882 1246.7224 -6870.0106 1396.3321 21739.583 -30424.594 298.96321 -90.586397 1.0254105 0.7025 -0.595439 0.2977195 + 600000 -5543.5677 1248.7682 -6792.3359 1346.9267 21827.244 -30425.377 299.45379 -291.40394 1.0195264 0.7 -0.59332 0.29666 + 605000 -5474.4972 1249.0604 -6723.5576 1386.9424 21886.849 -30420.093 299.52387 320.72375 1.0078262 0.6975 -0.591201 0.2956005 + 610000 -5444.4392 1268.064 -6712.5032 1336.2434 21909.556 -30420.625 304.08091 -498.11374 1.003753 0.695 -0.589082 0.294541 + 615000 -5576.1847 1305.9815 -6882.1662 1436.2497 21635.143 -30423.656 313.1735 278.60539 1.0176729 0.6925 -0.586963 0.2934815 + 620000 -5527.0897 1239.3078 -6766.3976 1371.3853 21850.583 -30424.469 297.1852 -80.897508 1.007611 0.69 -0.584844 0.292422 + 625000 -5552.9285 1232.0128 -6784.9412 1359.8043 21819.102 -30422.799 295.43585 -417.82041 1.0072509 0.6875 -0.582725 0.2913625 + 630000 -5532.1703 1252.0392 -6784.2094 1416.1466 21807.096 -30422.422 300.23817 369.4296 1.0039576 0.685 -0.580606 0.290303 + 635000 -5571.3255 1249.4908 -6820.8163 1369.6703 21808.738 -30421.932 299.62706 -297.44918 1.0196309 0.6825 -0.578487 0.2892435 + 640000 -5525.2478 1199.2506 -6724.4984 1369.7668 21894.477 -30421.099 287.5795 -72.546596 1.0007716 0.68 -0.576368 0.288184 + 645000 -5542.8568 1255.2316 -6798.0884 1331.6921 21842.315 -30419.358 301.0037 -814.5638 1.0097587 0.6775 -0.574249 0.2871245 + 650000 -5516.7418 1289.6956 -6806.4374 1400.2031 21783.51 -30421.477 309.26815 246.24345 1.0195846 0.675 -0.57213 0.286065 + 655000 -5524.8554 1265.5421 -6790.3975 1416.5196 21786.917 -30421.834 303.47617 529.12757 1.021072 0.6725 -0.570011 0.2850055 + 660000 -5460.2238 1295.8973 -6756.1212 1381.0945 21874.034 -30423.779 310.75533 569.85093 1.0270172 0.67 -0.567892 0.283946 + 665000 -5466.9896 1224.4909 -6691.4805 1377.5505 21903.453 -30415.905 293.63212 415.21587 1.0210181 0.6675 -0.565773 0.2828865 + 670000 -5508.3465 1292.0903 -6800.4368 1402.8444 21826.305 -30419.345 309.8424 362.60575 1.0200837 0.665 -0.563654 0.281827 + 675000 -5529.9825 1270.4695 -6800.452 1392.861 21763.662 -30422.666 304.65774 16.761021 1.0167408 0.6625 -0.561535 0.2807675 + 680000 -5589.5488 1256.7076 -6846.2564 1389.8603 21765.396 -30421.726 301.35765 -118.83353 1.0212456 0.66 -0.559416 0.279708 + 685000 -5461.9707 1282.6044 -6744.5751 1346.0454 21890.857 -30417.827 307.56769 -213.07775 1.0049252 0.6575 -0.557297 0.2786485 + 690000 -5591.4518 1229.1635 -6820.6153 1420.1767 21731.473 -30421.278 294.75259 194.31079 1.0204392 0.655 -0.555178 0.277589 + 695000 -5558.442 1216.2827 -6774.7247 1334.5019 21858.035 -30423.287 291.66379 -603.48558 1.0061448 0.6525 -0.553059 0.2765295 + 700000 -5492.653 1235.4799 -6728.1329 1391.2365 21871.21 -30420.123 296.26726 -41.586711 0.99217704 0.65 -0.55094 0.27547 + 705000 -5481.7595 1269.4684 -6751.2279 1323.5824 21897.228 -30413.262 304.4177 -459.11823 1.0141534 0.6475 -0.548821 0.2744105 + 710000 -5599.7613 1259.4323 -6859.1936 1383.1749 21741.135 -30418.84 302.01104 -358.23129 1.0156425 0.645 -0.546702 0.273351 + 715000 -5532.5713 1228.4966 -6761.0679 1396.4462 21859.914 -30416.53 294.59266 101.27196 1.0038296 0.6425 -0.544583 0.2722915 + 720000 -5584.0626 1229.4865 -6813.549 1388.5647 21769.406 -30418.339 294.83004 -792.67304 0.97380809 0.64 -0.542464 0.271232 + 725000 -5534.3566 1268.5686 -6802.9252 1423.0724 21758.618 -30416.045 304.20193 890.82213 1.0379062 0.6375 -0.540345 0.2701725 + 730000 -5578.7464 1212.5455 -6791.2919 1336.9104 21848.801 -30419.787 290.76761 -553.70126 1.0148611 0.635 -0.538226 0.269113 + 735000 -5618.1113 1206.2457 -6824.357 1375.0888 21801.81 -30417.694 289.25693 -427.22774 0.99982113 0.6325 -0.536107 0.2680535 + 740000 -5465.2032 1230.0118 -6695.215 1379.8548 21895.413 -30416.747 294.95602 315.63989 1.008153 0.63 -0.533988 0.266994 + 745000 -5578.5259 1240.851 -6819.3769 1426.7154 21759.96 -30418.763 297.55525 92.359506 0.99991361 0.6275 -0.531869 0.2659345 + 750000 -5534.8877 1256.4642 -6791.3518 1421.6831 21739.034 -30413.197 301.29928 158.88605 1.0081156 0.625 -0.52975 0.264875 + 755000 -5592.2413 1257.2462 -6849.4875 1366.9032 21783.562 -30418.46 301.48681 -581.61417 1.0056237 0.6225 -0.527631 0.2638155 + 760000 -5588.4063 1228.2425 -6816.6488 1410.3305 21756.183 -30416.341 294.53174 176.20865 1.011915 0.62 -0.525512 0.262756 + 765000 -5474.386 1218.7547 -6693.1407 1352.12 21939.502 -30416.343 292.25658 -320.6219 0.99100601 0.6175 -0.523393 0.2616965 + 770000 -5599.397 1242.7375 -6842.1345 1434.302 21734.26 -30418.21 298.00763 586.02998 1.0211757 0.615 -0.521274 0.260637 + 775000 -5508.8091 1258.5638 -6767.3729 1400.6493 21823.313 -30417.849 301.80278 178.09524 0.99995792 0.6125 -0.519155 0.2595775 + 780000 -5455.2192 1320.084 -6775.3032 1366.0858 21855.521 -30419.279 316.55527 160.2504 1.0264157 0.61 -0.517036 0.258518 + 785000 -5541.1905 1242.1565 -6783.347 1363.1625 21827.702 -30417.255 297.86832 -394.6575 1.0087294 0.6075 -0.514917 0.2574585 + 790000 -5569.3567 1256.7441 -6826.1008 1410.0504 21767.374 -30420.811 301.36641 -96.271799 0.99839497 0.605 -0.512798 0.256399 + 795000 -5537.502 1288.0636 -6825.5656 1435.9958 21711.599 -30417.456 308.8768 221.61169 1.0095005 0.6025 -0.510679 0.2553395 + 800000 -5560.8018 1240.0106 -6800.8124 1438.3265 21719.004 -30418.763 297.35373 775.89511 1.0301028 0.6 -0.50856 0.25428 + 805000 -5565.4599 1257.3617 -6822.8216 1394.5641 21770.009 -30412.251 301.5145 -173.96353 1.005747 0.5975 -0.506441 0.2532205 + 810000 -5497.4553 1284.5004 -6781.9556 1376.4375 21814.087 -30416.038 308.02234 188.25994 1.0214966 0.595 -0.504322 0.252161 + 815000 -5535.8991 1240.611 -6776.5101 1409.3101 21787.935 -30417.071 297.4977 262.65044 1.0117069 0.5925 -0.502203 0.2511015 + 820000 -5517.0729 1299.475 -6816.5479 1327.7472 21854.691 -30412.417 311.61325 -784.1981 1.0122843 0.59 -0.500084 0.250042 + 825000 -5632.9653 1192.4397 -6825.405 1360.7386 21778.67 -30413.486 285.94624 -388.42674 1.0232857 0.5875 -0.497965 0.2489825 + 830000 -5574.662 1257.8343 -6832.4962 1407.1117 21747.991 -30412.959 301.62783 -178.94106 1.0050031 0.585 -0.495846 0.247923 + 835000 -5517.9029 1249.5464 -6767.4493 1434.0858 21785.24 -30413.924 299.64041 516.05215 1.0102407 0.5825 -0.493727 0.2468635 + 840000 -5485.6365 1270.3474 -6755.9839 1335.1605 21840.353 -30411.626 304.62848 -421.00228 1.0133415 0.58 -0.491608 0.245804 + 845000 -5435.7784 1276.5836 -6712.362 1385.5475 21848.907 -30414.112 306.12391 556.14871 1.0239928 0.5775 -0.489489 0.2447445 + 850000 -5564.9093 1250.3893 -6815.2987 1376.9527 21812.84 -30414.994 299.84254 -103.38162 1.0245668 0.575 -0.48737 0.243685 + 855000 -5452.5901 1262.6663 -6715.2564 1378.7612 21881.921 -30415.305 302.78654 34.86207 0.99556351 0.5725 -0.485251 0.2426255 + 860000 -5515.222 1248.0439 -6763.2659 1305.4039 21931.869 -30415.835 299.28011 -908.54473 1.0020125 0.57 -0.483132 0.241566 + 865000 -5460.554 1245.0116 -6705.5656 1376.9247 21872.285 -30415.226 298.55297 -233.80476 0.98431622 0.5675 -0.481013 0.2405065 + 870000 -5476.9636 1272.0524 -6749.016 1362.2558 21845.497 -30416.364 305.03732 -99.216354 1.0114633 0.565 -0.478894 0.239447 + 875000 -5507.2163 1297.9312 -6805.1475 1404.1772 21725.275 -30415.229 311.24305 -162.72463 1.0065653 0.5625 -0.476775 0.2383875 + 880000 -5618.916 1185.2711 -6804.1871 1427.7925 21743.213 -30415.522 284.22724 324.75619 1.0135333 0.56 -0.474656 0.237328 + 885000 -5525.3698 1218.8864 -6744.2562 1360.4312 21880.194 -30413.898 292.28816 -244.38329 1.0135275 0.5575 -0.472537 0.2362685 + 890000 -5505.6369 1238.1666 -6743.8034 1412.5727 21840.193 -30413.202 296.91152 485.94408 1.0063342 0.555 -0.470418 0.235209 + 895000 -5602.1542 1221.0463 -6823.2005 1454.4394 21713.82 -30416.858 292.8061 268.08086 0.99556757 0.5525 -0.468299 0.2341495 + 900000 -5603.8654 1303.5628 -6907.4282 1499.2346 21573.659 -30417.214 312.59349 905.77809 1.0203813 0.55 -0.46618 0.23309 + 905000 -5561.5364 1235.1607 -6796.697 1317.7926 21863.665 -30411.262 296.19071 -899.61575 1.0112152 0.5475 -0.464061 0.2320305 + 910000 -5498.8354 1208.5542 -6707.3895 1394.7617 21874.374 -30410.334 289.81049 474.25045 1.0127648 0.545 -0.461942 0.230971 + 915000 -5592.5796 1194.943 -6787.5226 1393.8431 21788.893 -30415.252 286.54655 127.92132 1.0188573 0.5425 -0.459823 0.2299115 + 920000 -5496.6202 1245.2081 -6741.8283 1351.0512 21874.145 -30404.465 298.60007 2.9909425 1.0206197 0.54 -0.457704 0.228852 + 925000 -5491.3049 1264.9518 -6756.2567 1431.8079 21770.193 -30410.054 303.3346 495.87301 1.0083522 0.5375 -0.455585 0.2277925 + 930000 -5487.3571 1267.2843 -6754.6414 1386.5736 21833.656 -30414.507 303.89394 15.771272 1.0000236 0.535 -0.453466 0.226733 + 935000 -5476.9897 1259.4651 -6736.4548 1407.6301 21830.961 -30406.59 302.0189 581.34887 1.0155958 0.5325 -0.451347 0.2256735 + 940000 -5488.9916 1314.4567 -6803.4483 1466.6629 21737.432 -30413.446 315.20584 1238.1624 1.0288595 0.53 -0.449228 0.224614 + 945000 -5550.7311 1300.1216 -6850.8527 1412.3482 21723.824 -30411.146 311.76831 -169.42496 1.0064279 0.5275 -0.447109 0.2235545 + 950000 -5606.1286 1246.8667 -6852.9952 1472.7784 21676.262 -30413.203 298.9978 518.15276 1.0015579 0.525 -0.44499 0.222495 + 955000 -5539.4856 1253.2903 -6792.7759 1415.4535 21747.222 -30412.732 300.53818 480.02479 1.0216323 0.5225 -0.442871 0.2214355 + 960000 -5535.7958 1215.1899 -6750.9857 1431.2781 21814.097 -30411.881 291.40173 608.69976 1.0136976 0.52 -0.440752 0.220376 + 965000 -5576.559 1234.6144 -6811.1734 1514.023 21649.351 -30410.737 296.05972 1520.794 1.0106841 0.5175 -0.438633 0.2193165 + 970000 -5519.6002 1277.0163 -6796.6165 1466.0566 21699.821 -30415.026 306.22766 510.68622 0.98687386 0.515 -0.436514 0.218257 + 975000 -5591.0137 1254.588 -6845.6017 1429.5069 21702.359 -30411.235 300.84939 297.03602 1.0251394 0.5125 -0.434395 0.2171975 + 980000 -5540.9566 1234.8073 -6775.7639 1338.2556 21868.37 -30412.541 296.10597 -289.71499 1.0212154 0.51 -0.432276 0.216138 + 985000 -5511.5459 1300.1097 -6811.6556 1334.8282 21875.173 -30411.512 311.76544 -693.16807 1.0141634 0.5075 -0.430157 0.2150785 + 990000 -5516.8247 1251.5481 -6768.3728 1358.6891 21823.881 -30410.037 300.1204 -279.3824 1.005663 0.505 -0.428038 0.214019 + 995000 -5524.3561 1215.9313 -6740.2874 1390.687 21855.683 -30411.04 291.57953 -53.642919 0.99690457 0.5025 -0.425919 0.2129595 + 1000000 -5562.5466 1221.827 -6784.3736 1297.5519 21915.287 -30414.226 292.99331 -874.65244 1.0214982 0.5 -0.4238 0.2119 + 1005000 -5542.8861 1229.0938 -6771.9799 1429.8443 21760.776 -30409.275 294.73589 294.86058 0.99993056 0.4975 -0.421681 0.2108405 + 1010000 -5448.4956 1256.3749 -6704.8706 1306.8444 21971.623 -30407.793 301.27788 -942.92568 0.99248512 0.495 -0.419562 0.209781 + 1015000 -5485.7411 1220.3873 -6706.1284 1245.6819 22024.308 -30407.597 292.64806 -1773.327 0.98578512 0.4925 -0.417443 0.2087215 + 1020000 -5502.2773 1265.1614 -6767.4388 1300.0129 21841.931 -30407.973 303.38487 -1114.324 1.0096025 0.49 -0.415324 0.207662 + 1025000 -5494.1313 1283.1074 -6777.2387 1385.6888 21791.255 -30409.205 307.6883 338.12878 1.0286322 0.4875 -0.413205 0.2066025 + 1030000 -5489.9259 1292.1724 -6782.0983 1417.8757 21798.83 -30411.578 309.86209 146.08806 0.99796155 0.485 -0.411086 0.205543 + 1035000 -5580.7036 1227.9308 -6808.6343 1399.7311 21795.838 -30405.508 294.45698 245.3411 1.0203353 0.4825 -0.408967 0.2044835 + 1040000 -5614.6078 1226.5654 -6841.1731 1447.8292 21673.75 -30407.74 294.12956 291.24488 1.012026 0.48 -0.406848 0.203424 + 1045000 -5535.8226 1284.8577 -6820.6803 1375.0415 21807.843 -30408.339 308.10803 -154.07664 1.0167214 0.4775 -0.404729 0.2023645 + 1050000 -5534.4123 1250.095 -6784.5074 1287.9566 21899.002 -30411.016 299.77196 -1161.4357 1.0099009 0.475 -0.40261 0.201305 + 1055000 -5509.4752 1264.9106 -6774.3858 1424.5808 21771.718 -30409.977 303.32473 730.35249 1.0166966 0.4725 -0.400491 0.2002455 + 1060000 -5551.4539 1255.686 -6807.1399 1431.3525 21716.649 -30408.994 301.11268 339.03445 1.0030856 0.47 -0.398372 0.199186 + 1065000 -5539.4142 1262.7344 -6802.1485 1334.5366 21827.192 -30412.447 302.80287 -814.37495 1.0065153 0.4675 -0.396253 0.1981265 + 1070000 -5556.1691 1278.0638 -6834.2329 1448.2492 21710.563 -30409.035 306.47886 526.92483 1.0118442 0.465 -0.394134 0.197067 + 1075000 -5553.9187 1230.9623 -6784.881 1372.5373 21816.292 -30409.602 295.18396 -636.049 0.99842907 0.4625 -0.392015 0.1960075 + 1080000 -5486.3143 1233.146 -6719.4602 1265.7838 21983.124 -30408.372 295.70759 -1226.6108 1.0051532 0.46 -0.389896 0.194948 + 1085000 -5593.4276 1252.6224 -6846.0499 1398.5687 21750.307 -30408.85 300.37801 -199.06834 1.0113585 0.4575 -0.387777 0.1938885 + 1090000 -5615.0746 1189.5983 -6804.6728 1411.201 21758.699 -30408.722 285.26488 205.20475 1.0115124 0.455 -0.385658 0.192829 + 1095000 -5491.0496 1265.9512 -6757.0008 1285.0166 21903.564 -30410.304 303.57426 -1204.071 1.0039435 0.4525 -0.383539 0.1917695 + 1100000 -5602.5736 1259.8652 -6862.4388 1429.2182 21709.793 -30410.235 302.11484 -5.292079 1.0081307 0.45 -0.38142 0.19071 + 1105000 -5491.1776 1274.3708 -6765.5484 1363.4219 21817.395 -30405.818 305.59328 -273.46222 1.0048241 0.4475 -0.379301 0.1896505 + 1110000 -5591.6046 1226.4913 -6818.096 1386.3645 21774.994 -30407.268 294.11181 -54.656076 1.0128633 0.445 -0.377182 0.188591 + 1115000 -5471.7003 1250.9094 -6722.6097 1461.8829 21768.591 -30406.09 299.96726 1171.0124 1.0122551 0.4425 -0.375063 0.1875315 + 1120000 -5629.5757 1215.4906 -6845.0663 1433.0141 21685.611 -30403.695 291.47385 159.7385 1.011959 0.44 -0.372944 0.186472 + 1125000 -5609.7131 1243.3529 -6853.066 1394.1435 21715.512 -30408.244 298.15521 -613.37653 1.001762 0.4375 -0.370825 0.1854125 + 1130000 -5517.8437 1236.2289 -6754.0726 1381.5497 21865.453 -30407.635 296.44686 -50.670512 0.99784522 0.435 -0.368706 0.184353 + 1135000 -5484.0424 1262.4577 -6746.5001 1373.1642 21882.225 -30407.476 302.73652 -109.81081 0.99319158 0.4325 -0.366587 0.1832935 + 1140000 -5464.3225 1242.5082 -6706.8308 1353.1364 21902.474 -30406.855 297.95266 -193.64239 0.99369216 0.43 -0.364468 0.182234 + 1145000 -5616.5098 1231.4613 -6847.9711 1442.3498 21665.897 -30406.708 295.3036 197.4384 1.0013434 0.4275 -0.362349 0.1811745 + 1150000 -5570.9461 1220.6192 -6791.5654 1362.4483 21843.16 -30406.108 292.70369 -472.38468 0.99919513 0.425 -0.36023 0.180115 + 1155000 -5581.5228 1250.4929 -6832.0157 1399.2703 21713.872 -30407.442 299.86736 68.67313 1.0186094 0.4225 -0.358111 0.1790555 + 1160000 -5442.4593 1284.4383 -6726.8977 1424.4524 21823.753 -30403.289 308.00746 659.43865 1.0016333 0.42 -0.355992 0.177996 + 1165000 -5488.402 1266.3593 -6754.7613 1394.1319 21804.35 -30407.391 303.67212 327.86135 1.0142865 0.4175 -0.353873 0.1769365 + 1170000 -5476.149 1306.9735 -6783.1225 1429.0063 21755.183 -30406.784 313.41139 152.9988 0.9921559 0.415 -0.351754 0.175877 + 1175000 -5552.7195 1245.1137 -6797.8332 1460.5394 21718.862 -30405.892 298.57744 929.85903 1.0138017 0.4125 -0.349635 0.1748175 + 1180000 -5424.9372 1280.7552 -6705.6924 1363.0602 21952.781 -30406.655 307.12424 -15.436219 0.99469946 0.41 -0.347516 0.173758 + 1185000 -5547.6406 1244.8045 -6792.4451 1449.3378 21752.476 -30407.929 298.50331 1010.2202 1.029804 0.4075 -0.345397 0.1726985 + 1190000 -5518.33 1248.4316 -6766.7617 1416.6506 21831.584 -30406.854 299.37308 867.46971 1.0239472 0.405 -0.343278 0.171639 + 1195000 -5532.9651 1256.1747 -6789.1397 1330.4657 21843.722 -30406.423 301.22986 -733.82991 1.0016488 0.4025 -0.341159 0.1705795 + 1200000 -5553.066 1253.4742 -6806.5402 1351.6299 21838.577 -30408.246 300.58228 -305.68337 1.0122479 0.4 -0.33904 0.16952 + 1205000 -5519.1215 1249.3055 -6768.427 1350.1732 21804.158 -30403.344 299.58262 -543.0903 1.0018512 0.3975 -0.336921 0.1684605 + 1210000 -5527.9464 1276.42 -6804.3664 1482.4983 21659.348 -30403.541 306.08468 1352.4972 1.0313989 0.395 -0.334802 0.167401 + 1215000 -5528.3907 1207.7953 -6736.186 1320.6852 21903.038 -30405.73 289.62852 -718.49775 1.0053276 0.3925 -0.332683 0.1663415 + 1220000 -5530.8707 1284.7098 -6815.5805 1462.3884 21708.002 -30402.904 308.07255 591.73345 0.99594569 0.39 -0.330564 0.165282 + 1225000 -5510.4733 1242.8785 -6753.3518 1390.7837 21808.754 -30406.464 298.04145 414.48752 1.0266558 0.3875 -0.328445 0.1642225 + 1230000 -5511.1086 1256.014 -6767.1226 1411.3996 21786.166 -30400.32 301.19132 344.60627 1.0144412 0.385 -0.326326 0.163163 + 1235000 -5507.557 1263.7979 -6771.3549 1357.0612 21836.66 -30406.305 303.0579 -421.57464 0.99679054 0.3825 -0.324207 0.1621035 + 1240000 -5521.86 1251.2777 -6773.1376 1403.9886 21790.309 -30403.676 300.05556 223.64336 1.0138134 0.38 -0.322088 0.161044 + 1245000 -5495.1421 1282.2985 -6777.4406 1362.4685 21809.744 -30405.553 307.49434 -477.26223 1.0074571 0.3775 -0.319969 0.1599845 + 1250000 -5480.1139 1278.931 -6759.0449 1392.3278 21789.973 -30403.202 306.6868 -70.223368 0.99977242 0.375 -0.31785 0.158925 + 1255000 -5553.184 1252.6275 -6805.8115 1449.6591 21661.26 -30405.845 300.37925 667.59038 1.0204683 0.3725 -0.315731 0.1578655 + 1260000 -5500.3621 1303.3621 -6803.7242 1399.3677 21781.51 -30403.908 312.54537 -21.919882 1.0040001 0.37 -0.313612 0.156806 + 1265000 -5599.0593 1216.8118 -6815.8711 1448.5308 21734.171 -30405.807 291.79067 680.36892 1.0183911 0.3675 -0.311493 0.1557465 + 1270000 -5499.7199 1247.7042 -6747.4241 1385.0266 21829.215 -30405.427 299.19864 84.592133 0.99816992 0.365 -0.309374 0.154687 + 1275000 -5511.3148 1301.3431 -6812.658 1422.1524 21739.965 -30404.128 312.06122 680.64066 1.0347666 0.3625 -0.307255 0.1536275 + 1280000 -5548.0186 1270.302 -6818.3206 1334.2111 21757.058 -30406.305 304.61757 -967.15786 1.0109717 0.36 -0.305136 0.152568 + 1285000 -5555.0413 1280.8151 -6835.8564 1427.9067 21720.39 -30405.954 307.13862 368.48785 1.0080506 0.3575 -0.303017 0.1515085 + 1290000 -5496.7229 1283.2563 -6779.9793 1378.4038 21781.744 -30404.002 307.72402 8.3452567 1.0137024 0.355 -0.300898 0.150449 + 1295000 -5580.7272 1229.8436 -6810.5707 1366.8265 21783.805 -30404.506 294.91567 -460.30554 1.0085502 0.3525 -0.298779 0.1493895 + 1300000 -5538.7578 1235.394 -6774.1518 1436.4021 21779.128 -30402.492 296.24666 453.61372 0.99617343 0.35 -0.29666 0.14833 + 1305000 -5544.2 1218.136 -6762.336 1357.1673 21850.132 -30403.082 292.1082 -352.96744 1.0108396 0.3475 -0.294541 0.1472705 + 1310000 -5584.6337 1229.473 -6814.1067 1411.4248 21762.836 -30403.104 294.82682 394.62761 1.0211949 0.345 -0.292422 0.146211 + 1315000 -5436.0536 1293.1695 -6729.2231 1355.9564 21913.068 -30402.017 310.10119 -128.48391 1.0085461 0.3425 -0.290303 0.1451515 + 1320000 -5543.0105 1272.0501 -6815.0606 1405.9462 21736.124 -30402.088 305.03677 72.234346 1.0091715 0.34 -0.288184 0.144092 + 1325000 -5548.6163 1239.4869 -6788.1033 1407.6008 21777.171 -30402.102 297.22815 533.16549 1.0233863 0.3375 -0.286065 0.1430325 + 1330000 -5540.8843 1226.3995 -6767.2837 1501.7755 21677.206 -30403.423 294.08978 1240.633 1.0060816 0.335 -0.283946 0.141973 + 1335000 -5586.7627 1257.0633 -6843.826 1395.9593 21687.752 -30403.613 301.44296 -485.85005 1.0034879 0.3325 -0.281827 0.1409135 + 1340000 -5528.6391 1260.2811 -6788.9202 1285.0887 21886.122 -30404.215 302.21458 -1345.2852 1.0004527 0.33 -0.279708 0.139854 + 1345000 -5513.3113 1236.009 -6749.3203 1349.1797 21868.635 -30400.498 296.39414 -474.80145 1.0002406 0.3275 -0.277589 0.1387945 + 1350000 -5522.3768 1267.4323 -6789.8091 1507.5674 21607.325 -30402.928 303.92942 1773.9579 1.0298502 0.325 -0.27547 0.137735 + 1355000 -5551.6776 1188.9567 -6740.6343 1332.4533 21893.924 -30401.702 285.11104 -674.97023 1.0053334 0.3225 -0.273351 0.1366755 + 1360000 -5475.681 1262.222 -6737.903 1329.4781 21926.38 -30406.129 302.68 -200.38302 1.0156755 0.32 -0.271232 0.135616 + 1365000 -5509.3918 1284.2745 -6793.6663 1385.7835 21764.181 -30402.189 307.96817 -260.62121 0.99970234 0.3175 -0.269113 0.1345565 + 1370000 -5510.4413 1244.2221 -6754.6635 1394.1078 21834.473 -30402.98 298.36364 171.69348 1.0078346 0.315 -0.266994 0.133497 + 1375000 -5637.3565 1194.2641 -6831.6206 1352.6619 21790.919 -30403.683 286.38374 -926.31336 1.0014052 0.3125 -0.264875 0.1324375 + 1380000 -5535.2371 1268.7083 -6803.9454 1411.2035 21761.999 -30402.306 304.23541 1.7494611 1.0007021 0.31 -0.262756 0.131378 + 1385000 -5590.6929 1237.1763 -6827.8693 1425.1411 21700.436 -30401.792 296.67407 267.70563 1.0140025 0.3075 -0.260637 0.1303185 + 1390000 -5484.4968 1255.3342 -6739.831 1363.3349 21849.487 -30403.403 301.02831 -334.81743 0.9988136 0.305 -0.258518 0.129259 + 1395000 -5580.766 1216.9164 -6797.6824 1332.1573 21830.007 -30405.348 291.81574 -1230.1209 0.99078414 0.3025 -0.256399 0.1281995 + 1400000 -5542.6463 1270.5354 -6813.1816 1427.1867 21714.231 -30404.136 304.67354 455.15208 1.0157715 0.3 -0.25428 0.12714 + 1405000 -5466.7582 1275.6022 -6742.3604 1355.5736 21897.295 -30400.283 305.88856 -66.657731 1.0151471 0.2975 -0.252161 0.1260805 + 1410000 -5564.8474 1224.3922 -6789.2397 1330.6453 21814.374 -30400.427 293.60845 -774.92937 1.0136303 0.295 -0.250042 0.125021 + 1415000 -5430.2211 1308.0051 -6738.2263 1314.9611 21946.942 -30400.606 313.65877 -717.19161 1.0001463 0.2925 -0.247923 0.1239615 + 1420000 -5598.4838 1257.5011 -6855.9849 1415.0883 21711.443 -30404.958 301.54793 377.08012 1.0352276 0.29 -0.245804 0.122902 + 1425000 -5541.8937 1257.9576 -6799.8513 1400.0802 21773.418 -30403.035 301.6574 281.98161 1.0172007 0.2875 -0.243685 0.1218425 + 1430000 -5556.4346 1254.0053 -6810.4399 1407.8386 21766.868 -30402.715 300.70965 154.64262 1.0114215 0.285 -0.241566 0.120783 + 1435000 -5469.2552 1246.825 -6716.0802 1352.0012 21905.117 -30401.05 298.9878 -178.38426 1.0112743 0.2825 -0.239447 0.1197235 + 1440000 -5458.421 1254.9924 -6713.4134 1378.9303 21832.431 -30400.283 300.94636 -28.519956 0.99961051 0.28 -0.237328 0.118664 + 1445000 -5516.4714 1239.9243 -6756.3957 1405.0322 21790.978 -30400.325 297.33304 429.22692 1.0197884 0.2775 -0.235209 0.1176045 + 1450000 -5552.4255 1249.1142 -6801.5397 1361.7819 21785.875 -30399.912 299.53676 -769.48208 0.99105462 0.275 -0.23309 0.116545 + 1455000 -5517.8609 1220.2712 -6738.1321 1363.8136 21853.35 -30402.958 292.62024 -230.03249 1.0050729 0.2725 -0.230971 0.1154855 + 1460000 -5529.0421 1199.5637 -6728.6058 1281.1525 21947.817 -30402.254 287.65458 -1101.4619 1.0011538 0.27 -0.228852 0.114426 + 1465000 -5504.8124 1269.3051 -6774.1175 1397.0924 21778.739 -30400.882 304.37853 276.58705 1.0179163 0.2675 -0.226733 0.1133665 + 1470000 -5598.0732 1218.3941 -6816.4674 1418.8017 21718.631 -30401.044 292.17011 46.361297 1.0063797 0.265 -0.224614 0.112307 + 1475000 -5548.615 1254.4586 -6803.0737 1382.5152 21759.225 -30400 300.81835 -127.39812 1.0138223 0.2625 -0.222495 0.1112475 + 1480000 -5533.8343 1238.3003 -6772.1346 1436.9725 21759.67 -30400.105 296.94359 764.27767 1.0125907 0.26 -0.220376 0.110188 + 1485000 -5511.3105 1267.9695 -6779.28 1392.4351 21768.579 -30400.786 304.05824 -33.083281 1.008625 0.2575 -0.218257 0.1091285 + 1490000 -5492.7081 1249.7278 -6742.4359 1386.2265 21831.194 -30404.729 299.68391 167.62116 1.0065458 0.255 -0.216138 0.108069 + 1495000 -5565.5529 1257.6075 -6823.1605 1407.2006 21760.612 -30400.372 301.57346 -193.96975 0.99445457 0.2525 -0.214019 0.1070095 + 1500000 -5553.7271 1254.476 -6808.203 1411.4789 21739.068 -30400.776 300.82251 284.45877 1.0111814 0.25 -0.2119 0.10595 + 1505000 -5528.2919 1246.7404 -6775.0322 1494.6724 21668.724 -30400.534 298.96752 1503.5259 1.0199262 0.2475 -0.209781 0.1048905 + 1510000 -5529.2222 1256.6597 -6785.8818 1372.3032 21783.884 -30400.704 301.34616 -356.21245 1.0030809 0.245 -0.207662 0.103831 + 1515000 -5474.306 1292.5933 -6766.8993 1423.7446 21792.826 -30400.201 309.96302 458.75001 0.99603453 0.2425 -0.205543 0.1027715 + 1520000 -5533.15 1215.0608 -6748.2108 1345.4887 21859.918 -30402.082 291.37078 -574.91731 1.0075838 0.24 -0.203424 0.101712 + 1525000 -5534.3424 1275.3714 -6809.7138 1424.5986 21730.717 -30399.28 305.83323 403.09345 1.0170271 0.2375 -0.201305 0.1006525 + 1530000 -5594.0315 1228.3372 -6822.3687 1400.2671 21729.871 -30399.71 294.55445 -117.3201 1.0085693 0.235 -0.199186 0.099593 + 1535000 -5478.4978 1224.412 -6702.9097 1367.3472 21857.313 -30397.796 293.61318 -207.09591 1.0035875 0.2325 -0.197067 0.0985335 + 1540000 -5412.0571 1263.2997 -6675.3568 1365.2658 21913.322 -30400.612 302.93843 217.05159 1.0080531 0.23 -0.194948 0.097474 + 1545000 -5467.8087 1273.7408 -6741.5495 1392.9907 21826.459 -30393.776 305.44219 216.50192 1.0091255 0.2275 -0.192829 0.0964145 + 1550000 -5566.797 1224.1437 -6790.9407 1482.8048 21668.571 -30397.373 293.54885 729.69096 1.0040464 0.225 -0.19071 0.095355 + 1555000 -5497.5888 1259.6664 -6757.2552 1337.6561 21869.942 -30398.222 302.06718 -474.89321 1.0102649 0.2225 -0.188591 0.0942955 + 1560000 -5571.1541 1227.5401 -6798.6942 1443.2923 21702.982 -30401.053 294.36331 600.44478 1.0119193 0.22 -0.186472 0.093236 + 1565000 -5532.4484 1257.767 -6790.2154 1497.098 21657.763 -30398.914 301.61169 1582.7596 1.0279581 0.2175 -0.184353 0.0921765 + 1570000 -5491.3602 1250.8628 -6742.2231 1381.5334 21840.747 -30399 299.95608 319.87288 1.0172885 0.215 -0.182234 0.091117 + 1575000 -5448.4462 1268.2421 -6716.6883 1377.0235 21863.493 -30397.202 304.12362 -262.01917 0.98610142 0.2125 -0.180115 0.0900575 + 1580000 -5580.594 1226.2656 -6806.8595 1422.4858 21713.685 -30396.22 294.05768 357.25756 1.0113499 0.21 -0.177996 0.088998 + 1585000 -5541.0825 1278.2507 -6819.3331 1366.9139 21790.142 -30396.888 306.52367 -743.11524 0.98669546 0.2075 -0.175877 0.0879385 + 1590000 -5531.6035 1232.4292 -6764.0326 1395.465 21795.559 -30398.665 295.5357 42.444177 1.0126547 0.205 -0.173758 0.086879 + 1595000 -5461.4159 1287.0769 -6748.4928 1385.5573 21847.874 -30400.803 308.64018 618.18057 1.023846 0.2025 -0.171639 0.0858195 + 1600000 -5579.7572 1252.3883 -6832.1456 1403.3853 21764.717 -30403.163 300.3219 2.4209385 1.0090886 0.2 -0.16952 0.08476 + 1605000 -5491.6523 1248.6296 -6740.282 1420.4767 21788.29 -30399.543 299.42056 239.38793 0.99225234 0.1975 -0.167401 0.0837005 + 1610000 -5568.2821 1246.7876 -6815.0697 1370.1076 21752.385 -30399.233 298.97885 -468.73385 1.0081855 0.195 -0.165282 0.082641 + 1615000 -5530.1214 1224.1728 -6754.2942 1323.3884 21897.077 -30396.775 293.55582 -898.2112 0.99067847 0.1925 -0.163163 0.0815815 + 1620000 -5462.953 1253.936 -6716.889 1363.9087 21844.806 -30397.011 300.69303 49.460617 1.0189897 0.19 -0.161044 0.080522 + 1625000 -5568.5656 1231.6103 -6800.1759 1439.4406 21713.002 -30399.751 295.33933 590.45469 1.0150474 0.1875 -0.158925 0.0794625 + 1630000 -5513.0103 1230.3044 -6743.3148 1335.981 21895.338 -30397.086 295.02619 -710.37919 0.99994929 0.185 -0.156806 0.078403 + 1635000 -5580.5477 1231.076 -6811.6238 1376.4886 21798.044 -30400.814 295.21122 -206.8981 1.0081086 0.1825 -0.154687 0.0773435 + 1640000 -5540.3331 1256.5851 -6796.9181 1426.8142 21744.777 -30399.595 301.32827 745.57089 1.025877 0.18 -0.152568 0.076284 + 1645000 -5501.1298 1220.0238 -6721.1536 1389.073 21803.781 -30396.275 292.56091 131.20821 1.0089626 0.1775 -0.150449 0.0752245 + 1650000 -5628.1664 1220.696 -6848.8624 1362.9547 21731.445 -30400.636 292.7221 -616.61903 1.0201592 0.175 -0.14833 0.074165 + 1655000 -5592.5867 1211.2372 -6803.8239 1395.9179 21759.947 -30397.15 290.45388 18.913469 1.0103871 0.1725 -0.146211 0.0731055 + 1660000 -5428.2327 1278.6419 -6706.8747 1330.8879 21912.263 -30395.724 306.61749 -481.32774 0.99495838 0.17 -0.144092 0.072046 + 1665000 -5594.5998 1205.962 -6800.5618 1375.2659 21806.513 -30400.577 289.18889 -235.35107 1.0075357 0.1675 -0.141973 0.0709865 + 1670000 -5524.7971 1260.2713 -6785.0684 1344.6328 21851.941 -30396.671 302.21221 -544.14669 1.0114915 0.165 -0.139854 0.069927 + 1675000 -5468.1251 1274.7582 -6742.8834 1380.0088 21825.405 -30396.487 305.68618 38.686488 1.0120959 0.1625 -0.137735 0.0688675 + 1680000 -5473.7314 1242.9607 -6716.6921 1415.1549 21844.285 -30395.776 298.06116 641.25473 1.0114522 0.16 -0.135616 0.067808 + 1685000 -5636.5209 1211.507 -6848.0278 1459.7789 21659.328 -30400.565 290.51857 617.46049 1.0173888 0.1575 -0.133497 0.0667485 + 1690000 -5519.679 1240.1554 -6759.8344 1372.6554 21847.523 -30401.094 297.38845 -401.52343 0.99153655 0.155 -0.131378 0.065689 + 1695000 -5560.9528 1229.8902 -6790.843 1408.2004 21760.223 -30400.762 294.92687 100.67238 1.0033753 0.1525 -0.129259 0.0646295 + 1700000 -5554.1929 1255.6687 -6809.8616 1428.6747 21710.122 -30400.905 301.10854 159.06223 0.99158358 0.15 -0.12714 0.06357 + 1705000 -5542.8539 1230.6745 -6773.5285 1366.5362 21810.674 -30399.629 295.11494 -316.17067 0.99672752 0.1475 -0.125021 0.0625105 + 1710000 -5592.0441 1235.1898 -6827.2339 1407.2236 21724.16 -30397.816 296.19769 -171.98585 1.0010712 0.145 -0.122902 0.061451 + 1715000 -5456.289 1285.0566 -6741.3456 1366.6379 21844.445 -30394.859 308.15573 -103.28366 1.0059297 0.1425 -0.120783 0.0603915 + 1720000 -5492.3095 1254.7872 -6747.0967 1374.291 21873.54 -30397.737 300.89714 -10.335506 1.0122212 0.14 -0.118664 0.059332 + 1725000 -5518.9901 1240.2306 -6759.2207 1293.1765 21919.538 -30398.262 297.40649 -1112.3904 1.0060258 0.1375 -0.116545 0.0582725 + 1730000 -5461.6627 1250.9471 -6712.6098 1312.9461 21924.032 -30396.911 299.97629 -905.20686 0.99780548 0.135 -0.114426 0.057213 + 1735000 -5520.9229 1260.6697 -6781.5926 1351.3162 21851.369 -30398.186 302.30776 -380.34032 1.00967 0.1325 -0.112307 0.0561535 + 1740000 -5587.9319 1198.4018 -6786.3336 1377.0754 21811.513 -30398.588 287.37595 21.848644 1.019459 0.13 -0.110188 0.055094 + 1745000 -5548.2449 1238.1127 -6786.3576 1403.8402 21747.153 -30395.629 296.8986 135.03046 1.01201 0.1275 -0.108069 0.0540345 + 1750000 -5526.8784 1242.7978 -6769.6761 1309.8228 21913.235 -30398.337 298.02209 -850.35833 1.0059241 0.125 -0.10595 0.052975 + 1755000 -5508.0743 1308.7599 -6816.8341 1353.359 21760.125 -30400.191 313.83975 -749.9998 0.99685375 0.1225 -0.103831 0.0519155 + 1760000 -5542.8446 1265.3408 -6808.1854 1391.2186 21776.776 -30396.482 303.42789 117.90427 1.0236881 0.12 -0.101712 0.050856 + 1765000 -5526.6876 1268.5243 -6795.2119 1302.7664 21799.054 -30396.406 304.1913 -1161.5113 1.0216099 0.1175 -0.099593 0.0497965 + 1770000 -5579.5803 1246.8977 -6826.478 1486.5854 21651.454 -30399.954 299.00526 880.22447 1.0109541 0.115 -0.097474 0.048737 + 1775000 -5551.5053 1292.1271 -6843.6324 1405.0247 21688.07 -30398.863 309.85123 -251.03455 0.99873347 0.1125 -0.095355 0.0476775 + 1780000 -5515.8126 1282.9801 -6798.7927 1395.0164 21747.704 -30395.937 307.65779 333.80114 1.0239628 0.11 -0.093236 0.046618 + 1785000 -5510.7284 1208.8107 -6719.5391 1346.5399 21876.38 -30394.751 289.87201 -249.43776 1.0110945 0.1075 -0.091117 0.0455585 + 1790000 -5518.3081 1260.6355 -6778.9436 1399.7684 21796.94 -30397.3 302.29957 82.30008 1.0094344 0.105 -0.088998 0.044499 + 1795000 -5513.5389 1303.7533 -6817.2921 1421.97 21751.65 -30398.865 312.63917 500.03335 1.0096349 0.1025 -0.086879 0.0434395 + 1800000 -5572.7653 1214.8677 -6787.6331 1383.7533 21813.337 -30398.789 291.32448 -172.24736 1.0106323 0.1 -0.08476 0.04238 + 1805000 -5533.6069 1246.7853 -6780.3922 1412.9435 21800.334 -30394.707 298.97828 594.59696 1.0209456 0.0975 -0.082641 0.0413205 + 1810000 -5567.1324 1275.7954 -6842.9277 1442.9318 21676.766 -30398.215 305.93489 453.89649 1.0132054 0.095 -0.080522 0.040261 + 1815000 -5465.3294 1255.0662 -6720.3956 1341.0775 21912.58 -30394.54 300.96405 -302.59143 1.0016089 0.0925 -0.078403 0.0392015 + 1820000 -5588.4657 1233.5824 -6822.0481 1393.0967 21740.859 -30396.671 295.81224 -147.72631 1.0070758 0.09 -0.076284 0.038142 + 1825000 -5599.7235 1244.5645 -6844.288 1527.9294 21571.627 -30396.257 298.44576 1698.1117 1.0275341 0.0875 -0.074165 0.0370825 + 1830000 -5472.5833 1264.0019 -6736.5852 1327.0089 21908.977 -30394.568 303.10682 -394.05428 1.0135733 0.085 -0.072046 0.036023 + 1835000 -5559.5998 1243.4443 -6803.0442 1292.0778 21846.42 -30398.288 298.17713 -1254.4294 1.0141937 0.0825 -0.069927 0.0349635 + 1840000 -5498.5276 1283.2205 -6781.7481 1416.3308 21752.285 -30399.697 307.71542 317.53074 1.003693 0.08 -0.067808 0.033904 + 1845000 -5551.7395 1252.4882 -6804.2277 1374.7399 21772.215 -30396.489 300.34585 -263.50361 1.0114613 0.0775 -0.065689 0.0328445 + 1850000 -5525.5278 1240.7565 -6766.2843 1361.6208 21839.689 -30397.722 297.53259 -186.31575 1.0187838 0.075 -0.06357 0.031785 + 1855000 -5521.2867 1243.7856 -6765.0724 1347.4178 21874.625 -30396.108 298.25898 -284.75269 1.0228705 0.0725 -0.061451 0.0307255 + 1860000 -5640.4121 1146.9485 -6787.3606 1396.2408 21749.618 -30394.203 275.03749 -187.09544 1.006743 0.07 -0.059332 0.029666 + 1865000 -5525.2602 1260.9215 -6786.1817 1402.4348 21758.857 -30397.319 302.36813 -161.29785 0.99503244 0.0675 -0.057213 0.0286065 + 1870000 -5623.139 1216.967 -6840.1059 1413.5807 21703.495 -30397.576 291.82788 76.112007 1.0246491 0.065 -0.055094 0.027547 + 1875000 -5450.2615 1268.385 -6718.6465 1405.9759 21866.321 -30398.11 304.15789 307.93878 0.99773745 0.0625 -0.052975 0.0264875 + 1880000 -5539.8922 1245.1416 -6785.0337 1409.1823 21774.294 -30395.574 298.58413 82.127006 1.0058947 0.06 -0.050856 0.025428 + 1885000 -5514.3045 1240.5608 -6754.8653 1443.7995 21752.107 -30397.218 297.48566 611.17045 1.0085937 0.0575 -0.048737 0.0243685 + 1890000 -5438.3073 1258.3645 -6696.6718 1361.8049 21892.331 -30397.348 301.75497 -267.35927 0.9953092 0.055 -0.046618 0.023309 + 1895000 -5596.2731 1226.9718 -6823.2449 1402.9241 21732.185 -30395.165 294.22702 44.339776 1.0153331 0.0525 -0.044499 0.0222495 + 1900000 -5497.1413 1266.3676 -6763.5089 1373.9826 21832.991 -30399.356 303.67411 -27.290874 1.0132323 0.05 -0.04238 0.02119 + 1905000 -5517.4875 1236.6892 -6754.1767 1367.726 21857.698 -30393.874 296.55726 -499.76394 0.99284077 0.0475 -0.040261 0.0201305 + 1910000 -5552.0873 1269.7208 -6821.8081 1378.8021 21718.781 -30396.478 304.4782 -369.12045 1.0171861 0.045 -0.038142 0.019071 + 1915000 -5499.6451 1278.5051 -6778.1502 1430.6032 21725.761 -30396.795 306.58468 577.53122 1.01326 0.0425 -0.036023 0.0180115 + 1920000 -5487.2658 1251.9078 -6739.1736 1381.7635 21779.483 -30396.642 300.20666 -47.157489 1.0090288 0.04 -0.033904 0.016952 + 1925000 -5538.2174 1250.4348 -6788.6522 1378.1601 21776.002 -30397.01 299.85343 -635.20274 0.98963246 0.0375 -0.031785 0.0158925 + 1930000 -5539.4427 1260.8629 -6800.3057 1358.8021 21770.968 -30400.271 302.3541 -702.50699 0.99578271 0.035 -0.029666 0.014833 + 1935000 -5559.5408 1246.2889 -6805.8297 1426.6663 21737.226 -30396.429 298.85926 214.01347 1.0055051 0.0325 -0.027547 0.0137735 + 1940000 -5527.5821 1225.3364 -6752.9184 1378.8724 21850.255 -30395.169 293.83485 14.506752 1.0028753 0.03 -0.025428 0.012714 + 1945000 -5473.5551 1280.9318 -6754.4869 1425.106 21803.831 -30396.584 307.16659 506.19708 1.0032233 0.0275 -0.023309 0.0116545 + 1950000 -5567.2517 1222.2431 -6789.4948 1305.8232 21889.919 -30396.867 293.09308 -1294.1142 0.99788741 0.025 -0.02119 0.010595 + 1955000 -5484.5853 1260.6694 -6745.2547 1384.597 21831.404 -30392.929 302.30769 156.43697 1.0117059 0.0225 -0.019071 0.0095355 + 1960000 -5620.4544 1193.7438 -6814.1983 1383.8209 21783.257 -30401.598 286.25899 -66.845911 1.0220849 0.02 -0.016952 0.008476 + 1965000 -5545.5269 1233.4885 -6779.0154 1344.2005 21855.912 -30398.255 295.78973 -628.47314 1.0059777 0.0175 -0.014833 0.0074165 + 1970000 -5513.9042 1266.1527 -6780.057 1392.318 21762.753 -30395.542 303.62259 75.831811 1.0061881 0.015 -0.012714 0.006357 + 1975000 -5544.7648 1274.4083 -6819.1731 1391.4464 21746.624 -30396.103 305.60227 -75.461691 1.0034389 0.0125 -0.010595 0.0052975 + 1980000 -5538.5817 1248.0952 -6786.6769 1364.256 21830.009 -30395.903 299.2924 -83.978588 1.0205031 0.01 -0.008476 0.004238 + 1985000 -5595.8323 1230.885 -6826.7174 1357.6211 21780.506 -30393.427 295.16542 -709.11924 1.0035546 0.0075 -0.006357 0.0031785 + 1990000 -5482.4794 1286.6658 -6769.1452 1336.3899 21858.921 -30393.729 308.54161 -827.89365 1.0005728 0.005 -0.004238 0.002119 + 1995000 -5501.4886 1239.738 -6741.2266 1409.7478 21799.031 -30400.217 297.28836 653.89683 1.0202266 0.0025 -0.002119 0.0010595 + 2000000 -5611.4713 1229.0661 -6840.5373 1478.3644 21615.338 -30394.928 294.72923 480.97836 1.00402 0 -0 0 +Loop time of 13776.2 on 12 procs for 2000000 steps with 1800 atoms + +Performance: 12.543 ns/day, 1.913 hours/ns, 145.178 timesteps/s +95.3% CPU use with 12 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 7579.8 | 8070.7 | 8564.2 | 287.6 | 58.58 +Bond | 24.942 | 25.514 | 26.056 | 6.9 | 0.19 +Kspace | 2484.3 | 2979.3 | 3459.7 | 467.2 | 21.63 +Neigh | 523.25 | 524.15 | 525.06 | 2.5 | 3.80 +Comm | 806.39 | 872.63 | 948.24 | 194.3 | 6.33 +Output | 0.033634 | 0.034536 | 0.042227 | 1.2 | 0.00 +Modify | 864.36 | 1044.2 | 1191.5 | 438.5 | 7.58 +Other | | 259.6 | | | 1.88 + +Nlocal: 150.000 ave 158 max 141 min +Histogram: 1 0 0 2 2 3 2 1 0 1 +Nghost: 6106.00 ave 6170 max 6046 min +Histogram: 2 1 1 1 0 1 2 2 1 1 +Neighs: 86710.6 ave 92374 max 78975 min +Histogram: 1 0 0 3 0 3 1 0 2 2 + +Total # of neighbors = 1040527 +Ave neighs/atom = 578.07056 +Ave special neighs/atom = 2.0000000 +Neighbor list builds = 89189 +Dangerous builds = 0 +Total wall time: 4:00:09 diff --git a/examples/USER/fep/SPCEhyd/fep10/in-fep10-lj.lmp b/examples/USER/fep/SPCEhyd/fep10/in-fep10-lj.lmp new file mode 100644 index 0000000000..56d2994ec2 --- /dev/null +++ b/examples/USER/fep/SPCEhyd/fep10/in-fep10-lj.lmp @@ -0,0 +1,72 @@ +# created by fftool + +units real +boundary p p p + +atom_style full +bond_style harmonic +angle_style harmonic + +special_bonds lj/coul 0.0 0.0 0.5 + +# remove hybrid if not necessary +pair_style hybrid lj/cut/coul/long 12.0 12.0 lj/cut/soft 2 0.5 12.0 +pair_modify tail no +kspace_style pppm 1.0e-5 + +read_data data.lmp + +pair_coeff 1 1 lj/cut/soft 0.000000 1.000000 1.0 # Hwh Hwh +pair_coeff 1 2 lj/cut/soft 0.000000 1.000000 1.0 # Hwh Owh +pair_coeff 1 3 lj/cut/soft 0.000000 1.000000 1.0 # Hwh Hw +pair_coeff 1 4 lj/cut/soft 0.000000 1.000000 1.0 # Hwh Ow +pair_coeff 2 2 lj/cut/soft 0.155425 3.165500 1.0 # Owh Owh +pair_coeff 2 3 lj/cut/soft 0.000000 1.000000 1.0 # Owh Hw +pair_coeff 2 4 lj/cut/soft 0.155425 3.165500 1.0 # Owh Ow +pair_coeff 3 3 lj/cut/coul/long 0.000000 0.000000 # Hw Hw +pair_coeff 3 4 lj/cut/coul/long 0.000000 0.000000 # Hw Ow +pair_coeff 4 4 lj/cut/coul/long 0.155425 3.165500 # Ow Ow + +# minimize 1.0e-4 1.0e-6 100 1000 +# reset_timestep 0 + +fix SHAKE all shake 0.0001 20 0 b 1 + +neighbor 2.0 bin +# neigh_modify delay 0 every 1 check yes + +timestep 1.0 + +variable TK equal 300.0 +variable PBAR equal 1.0 + +velocity all create ${TK} 12345 + +fix TPSTAT all npt temp ${TK} ${TK} 100 iso ${PBAR} ${PBAR} 1000 + +thermo_style custom step cpu etotal ke pe evdwl ecoul elong temp press vol density +thermo 5000 + +set type 1*2 charge 0.0 + +run 100000 + +reset_timestep 0 + +variable lambda equal ramp(1.0,0.0) + +fix ADAPT all adapt/fep 100000 & + pair lj/cut/soft lambda 1*2 3*4 v_lambda & + after yes + +thermo_style custom step etotal ke pe evdwl ecoul elong temp press density v_lambda + +variable dlambda equal -0.05 + +compute FEP all fep ${TK} & + pair lj/cut/soft lambda 1*2 3*4 v_dlambda & + volume yes + +fix FEP all ave/time 20 4000 100000 c_FEP[*] file fep10-lj.fep + +run 2000000 diff --git a/examples/USER/fep/SPCEhyd/fep10/in-fep10-q.lmp b/examples/USER/fep/SPCEhyd/fep10/in-fep10-q.lmp new file mode 100644 index 0000000000..d9183f7f15 --- /dev/null +++ b/examples/USER/fep/SPCEhyd/fep10/in-fep10-q.lmp @@ -0,0 +1,76 @@ +# created by fftool + +units real +boundary p p p + +atom_style full +bond_style harmonic +angle_style harmonic + +special_bonds lj/coul 0.0 0.0 0.5 + +# remove hybrid if not necessary +pair_style lj/cut/coul/long 12.0 12.0 +pair_modify tail no +kspace_style pppm 1.0e-5 + +read_data data.lmp + +pair_coeff 1 1 0.000000 1.000000 # Hwh Hwh +pair_coeff 1 2 0.000000 1.000000 # Hwh Owh +pair_coeff 1 3 0.000000 1.000000 # Hwh Hw +pair_coeff 1 4 0.000000 1.000000 # Hwh Ow +pair_coeff 2 2 0.155425 3.165500 # Owh Owh +pair_coeff 2 3 0.000000 1.000000 # Owh Hw +pair_coeff 2 4 0.155425 3.165500 # Owh Ow +pair_coeff 3 3 0.000000 1.000000 # Hw Hw +pair_coeff 3 4 0.000000 1.000000 # Hw Ow +pair_coeff 4 4 0.155425 3.165500 # Ow Ow + +# minimize 1.0e-4 1.0e-6 100 1000 +# reset_timestep 0 + +fix SHAKE all shake 0.0001 20 0 b 1 + +neighbor 2.0 bin +# neigh_modify delay 0 every 1 check yes + +timestep 1.0 + +variable TK equal 300.0 +variable PBAR equal 1.0 + +velocity all create ${TK} 12345 + +fix TPSTAT all npt temp ${TK} ${TK} 100 iso ${PBAR} ${PBAR} 1000 + +thermo_style custom step cpu etotal ke pe evdwl ecoul elong temp press vol density +thermo 5000 + +run 100000 + +reset_timestep 0 + +variable lambda equal ramp(1.0,0.0) +variable qH equal 0.4238*v_lambda +variable qO equal -0.8476*v_lambda + +fix ADAPT all adapt/fep 100000 & + atom charge 1 v_qH & + atom charge 2 v_qO & + after yes + +thermo_style custom step etotal ke pe evdwl ecoul elong temp press density v_lambda v_qO v_qH + +variable dlambda equal -0.05 +variable dqH equal 0.4238*v_dlambda +variable dqO equal -0.8476*v_dlambda + +compute FEP all fep ${TK} & + atom charge 1 v_dqH & + atom charge 2 v_dqO & + volume yes + +fix FEP all ave/time 20 4000 100000 c_FEP[*] file fep10-q.fep + +run 2000000 diff --git a/examples/USER/fep/SPCEhyd/mols/data.lmp b/examples/USER/fep/SPCEhyd/mols/data.lmp new file mode 100644 index 0000000000..b6ead1d442 --- /dev/null +++ b/examples/USER/fep/SPCEhyd/mols/data.lmp @@ -0,0 +1,3635 @@ +created by fftool + +1800 atoms +1200 bonds +600 angles +4 atom types +1 bond types +1 angle types +0.000000 29.204526 xlo xhi +0.000000 29.204526 ylo yhi +0.000000 29.204526 zlo zhi + +Masses + + 1 1.008 # Hwh + 2 15.999 # Owh + 3 1.008 # Hw + 4 15.999 # Ow + +Bond Coeffs + + 1 517.630258 1.000000 # Ow-Hw + +Angle Coeffs + + 1 37.950526 109.470000 # Hw-Ow-Hw + +Atoms + + 1 1 1 0.423800 1.469233e+00 2.148142e+00 1.702038e+00 # Hwh SPCE + 2 1 2 -0.847600 1.979799e+00 1.409877e+00 2.142818e+00 # Owh SPCE + 3 1 1 0.423800 2.730865e+00 1.797090e+00 2.677576e+00 # Hwh SPCE + 4 2 3 0.423800 1.499900e+00 3.238303e+00 2.442531e+01 # Hw SPCE + 5 2 4 -0.847600 2.395752e+00 3.669948e+00 2.431980e+01 # Ow SPCE + 6 2 3 0.423800 2.296129e+00 4.663242e+00 2.437847e+01 # Hw SPCE + 7 3 3 0.423800 1.403661e+01 4.588770e+00 2.450878e+01 # Hw SPCE + 8 3 4 -0.847600 1.379951e+01 5.545591e+00 2.467695e+01 # Ow SPCE + 9 3 3 0.423800 1.402354e+01 6.091366e+00 2.386952e+01 # Hw SPCE + 10 4 3 0.423800 2.336049e+01 1.499786e+00 3.153124e+00 # Hw SPCE + 11 4 4 -0.847600 2.409385e+01 2.065977e+00 3.529439e+00 # Ow SPCE + 12 4 3 0.423800 2.490226e+01 1.500034e+00 3.691241e+00 # Hw SPCE + 13 5 3 0.423800 1.644196e+01 1.748367e+00 2.705436e+01 # Hw SPCE + 14 5 4 -0.847600 1.736546e+01 1.496062e+00 2.734329e+01 # Ow SPCE + 15 5 3 0.423800 1.783282e+01 2.301936e+00 2.770680e+01 # Hw SPCE + 16 6 3 0.423800 2.088933e+01 1.895579e+01 2.275309e+01 # Hw SPCE + 17 6 4 -0.847600 2.057929e+01 1.824793e+01 2.211841e+01 # Ow SPCE + 18 6 3 0.423800 1.984085e+01 1.772204e+01 2.254049e+01 # Hw SPCE + 19 7 3 0.423800 1.766189e+01 8.017800e+00 2.498964e+01 # Hw SPCE + 20 7 4 -0.847600 1.725196e+01 8.916174e+00 2.483189e+01 # Ow SPCE + 21 7 3 0.423800 1.625542e+01 8.836819e+00 2.485666e+01 # Hw SPCE + 22 8 3 0.423800 2.770687e+01 2.171554e+01 1.528009e+01 # Hw SPCE + 23 8 4 -0.847600 2.716811e+01 2.251048e+01 1.555905e+01 # Ow SPCE + 24 8 3 0.423800 2.770528e+01 2.307342e+01 1.618718e+01 # Hw SPCE + 25 9 3 0.423800 1.495385e+00 2.572800e+01 2.467279e+01 # Hw SPCE + 26 9 4 -0.847600 1.580382e+00 2.672428e+01 2.468743e+01 # Ow SPCE + 27 9 3 0.423800 1.767742e+00 2.702913e+01 2.562122e+01 # Hw SPCE + 28 10 3 0.423800 8.923646e+00 2.303176e+01 7.542360e+00 # Hw SPCE + 29 10 4 -0.847600 8.109007e+00 2.312152e+01 6.969379e+00 # Ow SPCE + 30 10 3 0.423800 7.298201e+00 2.288018e+01 7.502626e+00 # Hw SPCE + 31 11 3 0.423800 1.123659e+01 2.107791e+01 4.662100e+00 # Hw SPCE + 32 11 4 -0.847600 1.223538e+01 2.105512e+01 4.618509e+00 # Ow SPCE + 33 11 3 0.423800 1.259832e+01 2.069338e+01 5.477241e+00 # Hw SPCE + 34 12 3 0.423800 1.225298e+01 2.161218e+01 1.146502e+01 # Hw SPCE + 35 12 4 -0.847600 1.278626e+01 2.077538e+01 1.134100e+01 # Ow SPCE + 36 12 3 0.423800 1.216660e+01 1.999649e+01 1.124435e+01 # Hw SPCE + 37 13 3 0.423800 2.178085e+01 2.141393e+01 1.947311e+01 # Hw SPCE + 38 13 4 -0.847600 2.200950e+01 2.051108e+01 1.910897e+01 # Ow SPCE + 39 13 3 0.423800 2.190716e+01 1.982534e+01 1.982958e+01 # Hw SPCE + 40 14 3 0.423800 2.603847e+01 7.816317e+00 1.661274e+01 # Hw SPCE + 41 14 4 -0.847600 2.671903e+01 7.539282e+00 1.729103e+01 # Ow SPCE + 42 14 3 0.423800 2.712614e+01 6.667675e+00 1.701797e+01 # Hw SPCE + 43 15 3 0.423800 2.662104e+01 1.315276e+01 2.415551e+01 # Hw SPCE + 44 15 4 -0.847600 2.749092e+01 1.325361e+01 2.463837e+01 # Ow SPCE + 45 15 3 0.423800 2.770488e+01 1.422511e+01 2.474029e+01 # Hw SPCE + 46 16 3 0.423800 2.343696e+01 2.012869e+01 2.434140e+01 # Hw SPCE + 47 16 4 -0.847600 2.284311e+01 1.938463e+01 2.464753e+01 # Ow SPCE + 48 16 3 0.423800 2.328036e+01 1.889915e+01 2.540458e+01 # Hw SPCE + 49 17 3 0.423800 7.794521e+00 2.000989e+01 2.199182e+01 # Hw SPCE + 50 17 4 -0.847600 7.929378e+00 2.033080e+01 2.292928e+01 # Ow SPCE + 51 17 3 0.423800 8.857758e+00 2.010876e+01 2.322729e+01 # Hw SPCE + 52 18 3 0.423800 6.830410e+00 2.518010e+01 2.209866e+01 # Hw SPCE + 53 18 4 -0.847600 6.177519e+00 2.451264e+01 2.174056e+01 # Ow SPCE + 54 18 3 0.423800 6.309669e+00 2.363590e+01 2.220301e+01 # Hw SPCE + 55 19 3 0.423800 1.596377e+01 2.092461e+01 2.446023e+01 # Hw SPCE + 56 19 4 -0.847600 1.590738e+01 1.999342e+01 2.482037e+01 # Ow SPCE + 57 19 3 0.423800 1.681118e+01 1.956697e+01 2.478476e+01 # Hw SPCE + 58 20 3 0.423800 1.911326e+01 2.770546e+01 4.844631e+00 # Hw SPCE + 59 20 4 -0.847600 1.985915e+01 2.703939e+01 4.843586e+00 # Ow SPCE + 60 20 3 0.423800 2.073574e+01 2.752062e+01 4.840568e+00 # Hw SPCE + 61 21 3 0.423800 7.957677e+00 2.769262e+01 2.125162e+01 # Hw SPCE + 62 21 4 -0.847600 7.905132e+00 2.770650e+01 2.225015e+01 # Ow SPCE + 63 21 3 0.423800 8.829127e+00 2.770890e+01 2.263254e+01 # Hw SPCE + 64 22 3 0.423800 2.070265e+01 9.528473e+00 1.663276e+01 # Hw SPCE + 65 22 4 -0.847600 1.973165e+01 9.696078e+00 1.680328e+01 # Ow SPCE + 66 22 3 0.423800 1.934110e+01 8.919385e+00 1.729746e+01 # Hw SPCE + 67 23 3 0.423800 7.809959e+00 8.565464e+00 2.274346e+01 # Hw SPCE + 68 23 4 -0.847600 8.612896e+00 9.122442e+00 2.295575e+01 # Ow SPCE + 69 23 3 0.423800 9.372009e+00 8.852272e+00 2.236351e+01 # Hw SPCE + 70 24 3 0.423800 2.498899e+01 2.308095e+01 1.903531e+01 # Hw SPCE + 71 24 4 -0.847600 2.436019e+01 2.235497e+01 1.931382e+01 # Ow SPCE + 72 24 3 0.423800 2.484014e+01 2.170706e+01 1.990531e+01 # Hw SPCE + 73 25 3 0.423800 1.506895e+00 5.662694e+00 6.920599e+00 # Hw SPCE + 74 25 4 -0.847600 1.784503e+00 6.309796e+00 7.630665e+00 # Ow SPCE + 75 25 3 0.423800 2.707518e+00 6.641971e+00 7.436495e+00 # Hw SPCE + 76 26 3 0.423800 2.153403e+01 1.348159e+01 2.085837e+01 # Hw SPCE + 77 26 4 -0.847600 2.086248e+01 1.274077e+01 2.087218e+01 # Ow SPCE + 78 26 3 0.423800 2.067461e+01 1.247880e+01 2.181880e+01 # Hw SPCE + 79 27 3 0.423800 6.800685e+00 2.203104e+01 2.620499e+01 # Hw SPCE + 80 27 4 -0.847600 6.856841e+00 2.274258e+01 2.550459e+01 # Ow SPCE + 81 27 3 0.423800 6.369339e+00 2.244243e+01 2.468469e+01 # Hw SPCE + 82 28 3 0.423800 2.330650e+01 7.514481e+00 5.018724e+00 # Hw SPCE + 83 28 4 -0.847600 2.253073e+01 6.885276e+00 4.970822e+00 # Ow SPCE + 84 28 3 0.423800 2.286694e+01 5.944773e+00 4.921621e+00 # Hw SPCE + 85 29 3 0.423800 1.465322e+01 1.590735e+01 2.422856e+01 # Hw SPCE + 86 29 4 -0.847600 1.477140e+01 1.663018e+01 2.354771e+01 # Ow SPCE + 87 29 3 0.423800 1.453616e+01 1.751291e+01 2.395449e+01 # Hw SPCE + 88 30 3 0.423800 1.040291e+01 5.254847e+00 1.370622e+01 # Hw SPCE + 89 30 4 -0.847600 1.105724e+01 4.607844e+00 1.409768e+01 # Ow SPCE + 90 30 3 0.423800 1.077812e+01 3.674305e+00 1.387274e+01 # Hw SPCE + 91 31 3 0.423800 2.276938e+01 1.512055e+01 9.190561e+00 # Hw SPCE + 92 31 4 -0.847600 2.301556e+01 1.453653e+01 9.964062e+00 # Ow SPCE + 93 31 3 0.423800 2.340966e+01 1.368242e+01 9.624659e+00 # Hw SPCE + 94 32 3 0.423800 5.188719e+00 2.026958e+01 2.114125e+01 # Hw SPCE + 95 32 4 -0.847600 5.223057e+00 2.126505e+01 2.105256e+01 # Ow SPCE + 96 32 3 0.423800 4.292242e+00 2.162905e+01 2.101958e+01 # Hw SPCE + 97 33 3 0.423800 1.776246e+01 1.332125e+01 2.630655e+01 # Hw SPCE + 98 33 4 -0.847600 1.826497e+01 1.415453e+01 2.653702e+01 # Ow SPCE + 99 33 3 0.423800 1.924234e+01 1.400708e+01 2.638536e+01 # Hw SPCE + 100 34 3 0.423800 2.606566e+00 1.054762e+01 6.428992e+00 # Hw SPCE + 101 34 4 -0.847600 1.625220e+00 1.071787e+01 6.339693e+00 # Ow SPCE + 102 34 3 0.423800 1.425906e+00 1.104165e+01 5.414793e+00 # Hw SPCE + 103 35 3 0.423800 2.588493e+01 4.071742e+00 4.105367e+00 # Hw SPCE + 104 35 4 -0.847600 2.671121e+01 4.617835e+00 4.243387e+00 # Ow SPCE + 105 35 3 0.423800 2.741817e+01 4.051459e+00 4.666975e+00 # Hw SPCE + 106 36 3 0.423800 7.373843e+00 6.669901e+00 1.394201e+01 # Hw SPCE + 107 36 4 -0.847600 7.563042e+00 6.090000e+00 1.473442e+01 # Ow SPCE + 108 36 3 0.423800 7.053521e+00 6.429426e+00 1.552510e+01 # Hw SPCE + 109 37 3 0.423800 4.172482e+00 1.303888e+01 1.710824e+01 # Hw SPCE + 110 37 4 -0.847600 4.702018e+00 1.220690e+01 1.727375e+01 # Ow SPCE + 111 37 3 0.423800 4.203212e+00 1.141758e+01 1.691575e+01 # Hw SPCE + 112 38 3 0.423800 1.949787e+01 5.529973e+00 1.254239e+01 # Hw SPCE + 113 38 4 -0.847600 2.035077e+01 5.090120e+00 1.282362e+01 # Ow SPCE + 114 38 3 0.423800 2.089205e+01 4.864082e+00 1.201373e+01 # Hw SPCE + 115 39 3 0.423800 2.451616e+01 8.924670e+00 1.865568e+01 # Hw SPCE + 116 39 4 -0.847600 2.381451e+01 8.652982e+00 1.799699e+01 # Ow SPCE + 117 39 3 0.423800 2.311781e+01 9.367935e+00 1.793821e+01 # Hw SPCE + 118 40 3 0.423800 1.500065e+00 2.219167e+01 2.760505e+00 # Hw SPCE + 119 40 4 -0.847600 2.224501e+00 2.150236e+01 2.767217e+00 # Ow SPCE + 120 40 3 0.423800 1.817068e+00 2.059122e+01 2.829029e+00 # Hw SPCE + 121 41 3 0.423800 2.640554e+01 1.928714e+01 4.290644e+00 # Hw SPCE + 122 41 4 -0.847600 2.740415e+01 1.931798e+01 4.247967e+00 # Ow SPCE + 123 41 3 0.423800 2.770519e+01 2.026827e+01 4.168519e+00 # Hw SPCE + 124 42 3 0.423800 1.388747e+01 7.977813e+00 2.595555e+01 # Hw SPCE + 125 42 4 -0.847600 1.290644e+01 7.828209e+00 2.607882e+01 # Ow SPCE + 126 42 3 0.423800 1.244102e+01 8.710473e+00 2.614947e+01 # Hw SPCE + 127 43 3 0.423800 2.444541e+01 1.183745e+01 2.520365e+01 # Hw SPCE + 128 43 4 -0.847600 2.466934e+01 1.086365e+01 2.516413e+01 # Ow SPCE + 129 43 3 0.423800 2.566048e+01 1.075222e+01 2.509189e+01 # Hw SPCE + 130 44 3 0.423800 2.483417e+01 1.651695e+01 2.163120e+01 # Hw SPCE + 131 44 4 -0.847600 2.410271e+01 1.585943e+01 2.145058e+01 # Ow SPCE + 132 44 3 0.423800 2.445847e+01 1.492995e+01 2.154809e+01 # Hw SPCE + 133 45 3 0.423800 1.730263e+01 2.595540e+01 3.740298e+00 # Hw SPCE + 134 45 4 -0.847600 1.773679e+01 2.510003e+01 4.022883e+00 # Ow SPCE + 135 45 3 0.423800 1.770998e+01 2.444677e+01 3.266223e+00 # Hw SPCE + 136 46 3 0.423800 1.064666e+01 1.544703e+00 2.342635e+01 # Hw SPCE + 137 46 4 -0.847600 9.658303e+00 1.485990e+00 2.356670e+01 # Ow SPCE + 138 46 3 0.423800 9.290506e+00 2.400694e+00 2.373415e+01 # Hw SPCE + 139 47 3 0.423800 6.958351e+00 1.058272e+01 2.770522e+01 # Hw SPCE + 140 47 4 -0.847600 7.092574e+00 1.157367e+01 2.770263e+01 # Ow SPCE + 141 47 3 0.423800 6.203030e+00 1.203050e+01 2.769941e+01 # Hw SPCE + 142 48 3 0.423800 3.958262e+00 4.358392e+00 8.337038e+00 # Hw SPCE + 143 48 4 -0.847600 3.059091e+00 3.923448e+00 8.385147e+00 # Ow SPCE + 144 48 3 0.423800 3.110013e+00 3.117001e+00 8.974257e+00 # Hw SPCE + 145 49 3 0.423800 2.527221e+01 1.981465e+01 9.871580e+00 # Hw SPCE + 146 49 4 -0.847600 2.589272e+01 1.903608e+01 9.965427e+00 # Ow SPCE + 147 49 3 0.423800 2.675477e+01 1.934357e+01 1.036832e+01 # Hw SPCE + 148 50 3 0.423800 2.513678e+01 1.655111e+01 8.455263e+00 # Hw SPCE + 149 50 4 -0.847600 2.439311e+01 1.719386e+01 8.271354e+00 # Ow SPCE + 150 50 3 0.423800 2.390732e+01 1.691348e+01 7.443466e+00 # Hw SPCE + 151 51 3 0.423800 9.547080e+00 5.818685e+00 7.837619e+00 # Hw SPCE + 152 51 4 -0.847600 9.188230e+00 4.885428e+00 7.821533e+00 # Ow SPCE + 153 51 3 0.423800 9.947239e+00 4.237539e+00 7.757147e+00 # Hw SPCE + 154 52 3 0.423800 1.913212e+01 1.999545e+01 1.375657e+01 # Hw SPCE + 155 52 4 -0.847600 1.977845e+01 1.954444e+01 1.437208e+01 # Ow SPCE + 156 52 3 0.423800 2.009170e+01 1.868964e+01 1.395831e+01 # Hw SPCE + 157 53 3 0.423800 2.488521e+00 4.298268e+00 1.662022e+01 # Hw SPCE + 158 53 4 -0.847600 3.439105e+00 4.019650e+00 1.675720e+01 # Ow SPCE + 159 53 3 0.423800 3.840963e+00 4.558416e+00 1.749763e+01 # Hw SPCE + 160 54 3 0.423800 8.720856e+00 1.200478e+01 2.905857e+00 # Hw SPCE + 161 54 4 -0.847600 8.109727e+00 1.255626e+01 2.338060e+00 # Ow SPCE + 162 54 3 0.423800 7.159796e+00 1.233215e+01 2.555793e+00 # Hw SPCE + 163 55 3 0.423800 1.270857e+01 1.035303e+01 5.382569e+00 # Hw SPCE + 164 55 4 -0.847600 1.322488e+01 9.496640e+00 5.388255e+00 # Ow SPCE + 165 55 3 0.423800 1.305339e+01 9.009733e+00 6.244710e+00 # Hw SPCE + 166 56 3 0.423800 1.684898e+01 2.523462e+01 1.822162e+01 # Hw SPCE + 167 56 4 -0.847600 1.664347e+01 2.603725e+01 1.878158e+01 # Ow SPCE + 168 56 3 0.423800 1.648466e+01 2.575236e+01 1.972689e+01 # Hw SPCE + 169 57 3 0.423800 1.020796e+01 2.594114e+01 7.868994e+00 # Hw SPCE + 170 57 4 -0.847600 1.048594e+01 2.681088e+01 8.276773e+00 # Ow SPCE + 171 57 3 0.423800 1.003609e+01 2.692243e+01 9.162888e+00 # Hw SPCE + 172 58 3 0.423800 2.300209e+01 1.144525e+01 1.973611e+01 # Hw SPCE + 173 58 4 -0.847600 2.326720e+01 1.069841e+01 2.034599e+01 # Ow SPCE + 174 58 3 0.423800 2.274079e+01 1.075787e+01 2.119414e+01 # Hw SPCE + 175 59 3 0.423800 1.609161e+01 1.214359e+01 1.492237e+00 # Hw SPCE + 176 59 4 -0.847600 1.690824e+01 1.157531e+01 1.593113e+00 # Ow SPCE + 177 59 3 0.423800 1.664177e+01 1.065885e+01 1.891605e+00 # Hw SPCE + 178 60 3 0.423800 2.419902e+01 2.411706e+01 6.611761e+00 # Hw SPCE + 179 60 4 -0.847600 2.467864e+01 2.498895e+01 6.710596e+00 # Ow SPCE + 180 60 3 0.423800 2.544873e+01 2.487644e+01 7.338537e+00 # Hw SPCE + 181 61 3 0.423800 1.196568e+01 5.594696e+00 6.473038e+00 # Hw SPCE + 182 61 4 -0.847600 1.199305e+01 6.536579e+00 6.807864e+00 # Ow SPCE + 183 61 3 0.423800 1.267510e+01 6.612062e+00 7.535263e+00 # Hw SPCE + 184 62 3 0.423800 1.669021e+01 8.472310e+00 7.036663e+00 # Hw SPCE + 185 62 4 -0.847600 1.572284e+01 8.690729e+00 6.908251e+00 # Ow SPCE + 186 62 3 0.423800 1.554585e+01 9.621300e+00 7.228737e+00 # Hw SPCE + 187 63 3 0.423800 1.821528e+01 1.487703e+01 1.756524e+01 # Hw SPCE + 188 63 4 -0.847600 1.872911e+01 1.551055e+01 1.814370e+01 # Ow SPCE + 189 63 3 0.423800 1.854708e+01 1.644985e+01 1.785290e+01 # Hw SPCE + 190 64 3 0.423800 7.984302e+00 1.799893e+01 1.529353e+00 # Hw SPCE + 191 64 4 -0.847600 8.826448e+00 1.746123e+01 1.488465e+00 # Ow SPCE + 192 64 3 0.423800 9.462122e+00 1.778360e+01 2.189889e+00 # Hw SPCE + 193 65 3 0.423800 2.562243e+01 6.741416e+00 1.403530e+01 # Hw SPCE + 194 65 4 -0.847600 2.622983e+01 6.086036e+00 1.448423e+01 # Ow SPCE + 195 65 3 0.423800 2.717413e+01 6.262256e+00 1.420632e+01 # Hw SPCE + 196 66 3 0.423800 2.512713e+01 2.094793e+00 1.358574e+01 # Hw SPCE + 197 66 4 -0.847600 2.434949e+01 1.495442e+00 1.339585e+01 # Ow SPCE + 198 66 3 0.423800 2.355565e+01 1.803117e+00 1.392040e+01 # Hw SPCE + 199 67 3 0.423800 9.301908e+00 1.331503e+01 2.358584e+01 # Hw SPCE + 200 67 4 -0.847600 8.584183e+00 1.395669e+01 2.331540e+01 # Ow SPCE + 201 67 3 0.423800 8.271122e+00 1.373659e+01 2.239153e+01 # Hw SPCE + 202 68 3 0.423800 9.955846e+00 1.273829e+01 6.296276e+00 # Hw SPCE + 203 68 4 -0.847600 1.056341e+01 1.199023e+01 6.029323e+00 # Ow SPCE + 204 68 3 0.423800 1.029850e+01 1.165171e+01 5.126425e+00 # Hw SPCE + 205 69 3 0.423800 1.595831e+01 4.015404e+00 2.177868e+01 # Hw SPCE + 206 69 4 -0.847600 1.665439e+01 3.881186e+00 2.248398e+01 # Ow SPCE + 207 69 3 0.423800 1.647833e+01 4.501446e+00 2.324836e+01 # Hw SPCE + 208 70 3 0.423800 1.191249e+01 1.053961e+01 1.614692e+01 # Hw SPCE + 209 70 4 -0.847600 1.285965e+01 1.031522e+01 1.591770e+01 # Ow SPCE + 210 70 3 0.423800 1.326240e+01 1.106545e+01 1.539335e+01 # Hw SPCE + 211 71 3 0.423800 2.756106e+01 7.219592e+00 3.733621e+00 # Hw SPCE + 212 71 4 -0.847600 2.673030e+01 7.191784e+00 3.177694e+00 # Ow SPCE + 213 71 3 0.423800 2.592863e+01 7.212180e+00 3.775112e+00 # Hw SPCE + 214 72 3 0.423800 1.619051e+01 4.555961e+00 1.375257e+01 # Hw SPCE + 215 72 4 -0.847600 1.693660e+01 4.702954e+00 1.310315e+01 # Ow SPCE + 216 72 3 0.423800 1.692272e+01 3.981627e+00 1.241069e+01 # Hw SPCE + 217 73 3 0.423800 2.233085e+01 2.254376e+01 2.509491e+01 # Hw SPCE + 218 73 4 -0.847600 2.137546e+01 2.283349e+01 2.503760e+01 # Ow SPCE + 219 73 3 0.423800 2.132827e+01 2.383218e+01 2.505737e+01 # Hw SPCE + 220 74 3 0.423800 4.614385e+00 2.266996e+01 1.754496e+01 # Hw SPCE + 221 74 4 -0.847600 5.529231e+00 2.271822e+01 1.794587e+01 # Ow SPCE + 222 74 3 0.423800 6.044304e+00 2.345795e+01 1.751287e+01 # Hw SPCE + 223 75 3 0.423800 2.221903e+01 7.556525e+00 1.516871e+01 # Hw SPCE + 224 75 4 -0.847600 2.250349e+01 7.635346e+00 1.421326e+01 # Ow SPCE + 225 75 3 0.423800 2.176672e+01 7.313720e+00 1.361852e+01 # Hw SPCE + 226 76 3 0.423800 2.629759e+01 1.920638e+01 2.353714e+01 # Hw SPCE + 227 76 4 -0.847600 2.715395e+01 1.902228e+01 2.401958e+01 # Ow SPCE + 228 76 3 0.423800 2.770506e+01 1.837985e+01 2.348707e+01 # Hw SPCE + 229 77 3 0.423800 4.143809e+00 1.561555e+01 8.018737e+00 # Hw SPCE + 230 77 4 -0.847600 4.614959e+00 1.553180e+01 8.896805e+00 # Ow SPCE + 231 77 3 0.423800 4.199222e+00 1.615528e+01 9.558946e+00 # Hw SPCE + 232 78 3 0.423800 9.954198e+00 1.261788e+01 1.237010e+01 # Hw SPCE + 233 78 4 -0.847600 9.562880e+00 1.353598e+01 1.243307e+01 # Ow SPCE + 234 78 3 0.423800 8.569485e+00 1.346921e+01 1.252638e+01 # Hw SPCE + 235 79 3 0.423800 2.332846e+01 2.323923e+01 3.781581e+00 # Hw SPCE + 236 79 4 -0.847600 2.276687e+01 2.256928e+01 3.296002e+00 # Ow SPCE + 237 79 3 0.423800 2.333089e+01 2.178393e+01 3.040806e+00 # Hw SPCE + 238 80 3 0.423800 1.563376e+01 9.681389e+00 1.524542e+01 # Hw SPCE + 239 80 4 -0.847600 1.641808e+01 9.172361e+00 1.559999e+01 # Ow SPCE + 240 80 3 0.423800 1.655877e+01 9.404722e+00 1.656239e+01 # Hw SPCE + 241 81 3 0.423800 3.016047e+00 1.608090e+01 2.770691e+01 # Hw SPCE + 242 81 4 -0.847600 3.873141e+00 1.658768e+01 2.761432e+01 # Ow SPCE + 243 81 3 0.423800 4.041570e+00 1.678405e+01 2.664837e+01 # Hw SPCE + 244 82 3 0.423800 2.553862e+01 1.927894e+01 1.283389e+01 # Hw SPCE + 245 82 4 -0.847600 2.595825e+01 2.010455e+01 1.321108e+01 # Ow SPCE + 246 82 3 0.423800 2.694466e+01 1.996635e+01 1.329986e+01 # Hw SPCE + 247 83 3 0.423800 1.446078e+01 5.122478e+00 1.975466e+01 # Hw SPCE + 248 83 4 -0.847600 1.399157e+01 5.719286e+00 1.910377e+01 # Ow SPCE + 249 83 3 0.423800 1.387428e+01 6.626403e+00 1.950797e+01 # Hw SPCE + 250 84 3 0.423800 4.975267e+00 1.195188e+01 1.361200e+01 # Hw SPCE + 251 84 4 -0.847600 5.863360e+00 1.223599e+01 1.397335e+01 # Ow SPCE + 252 84 3 0.423800 6.001585e+00 1.182880e+01 1.487617e+01 # Hw SPCE + 253 85 3 0.423800 1.729394e+01 2.690297e+01 2.489413e+01 # Hw SPCE + 254 85 4 -0.847600 1.798622e+01 2.618717e+01 2.498570e+01 # Ow SPCE + 255 85 3 0.423800 1.753851e+01 2.531197e+01 2.516899e+01 # Hw SPCE + 256 86 3 0.423800 1.443295e+01 1.369845e+01 9.542794e+00 # Hw SPCE + 257 86 4 -0.847600 1.453225e+01 1.337010e+01 1.048212e+01 # Ow SPCE + 258 86 3 0.423800 1.407152e+01 1.248767e+01 1.057721e+01 # Hw SPCE + 259 87 3 0.423800 1.481041e+01 9.892914e+00 2.769929e+01 # Hw SPCE + 260 87 4 -0.847600 1.474545e+01 1.089078e+01 2.770556e+01 # Ow SPCE + 261 87 3 0.423800 1.566461e+01 1.128460e+01 2.771213e+01 # Hw SPCE + 262 88 3 0.423800 2.711944e+01 2.239741e+00 7.551846e+00 # Hw SPCE + 263 88 4 -0.847600 2.613715e+01 2.417616e+00 7.610794e+00 # Ow SPCE + 264 88 3 0.423800 2.588056e+01 2.557524e+00 8.567132e+00 # Hw SPCE + 265 89 3 0.423800 1.579109e+01 5.900382e+00 6.668995e+00 # Hw SPCE + 266 89 4 -0.847600 1.499190e+01 6.158630e+00 6.126217e+00 # Ow SPCE + 267 89 3 0.423800 1.487648e+01 5.510338e+00 5.373624e+00 # Hw SPCE + 268 90 3 0.423800 2.133610e+01 1.746297e+01 9.335501e+00 # Hw SPCE + 269 90 4 -0.847600 2.057506e+01 1.754198e+01 8.691632e+00 # Ow SPCE + 270 90 3 0.423800 2.092169e+01 1.747499e+01 7.756026e+00 # Hw SPCE + 271 91 3 0.423800 5.764508e+00 6.459756e+00 1.799860e+01 # Hw SPCE + 272 91 4 -0.847600 6.332363e+00 5.967190e+00 1.865808e+01 # Ow SPCE + 273 91 3 0.423800 6.766765e+00 6.620913e+00 1.927772e+01 # Hw SPCE + 274 92 3 0.423800 1.519730e+01 1.983322e+01 1.921450e+01 # Hw SPCE + 275 92 4 -0.847600 1.593017e+01 1.916543e+01 1.908429e+01 # Ow SPCE + 276 92 3 0.423800 1.663520e+01 1.955553e+01 1.849204e+01 # Hw SPCE + 277 93 3 0.423800 2.313253e+00 7.250692e+00 1.329362e+01 # Hw SPCE + 278 93 4 -0.847600 1.488319e+00 7.811762e+00 1.322518e+01 # Ow SPCE + 279 93 3 0.423800 1.540991e+00 8.563456e+00 1.388259e+01 # Hw SPCE + 280 94 3 0.423800 2.611731e+01 1.202552e+01 1.053768e+01 # Hw SPCE + 281 94 4 -0.847600 2.710862e+01 1.215596e+01 1.052099e+01 # Ow SPCE + 282 94 3 0.423800 2.756049e+01 1.126726e+01 1.044331e+01 # Hw SPCE + 283 95 3 0.423800 6.603696e+00 2.054040e+01 1.298798e+01 # Hw SPCE + 284 95 4 -0.847600 5.882293e+00 2.028304e+01 1.234506e+01 # Ow SPCE + 285 95 3 0.423800 6.172577e+00 2.050160e+01 1.141341e+01 # Hw SPCE + 286 96 3 0.423800 2.098737e+01 6.874062e+00 1.013861e+01 # Hw SPCE + 287 96 4 -0.847600 2.156028e+01 7.451122e+00 1.072066e+01 # Ow SPCE + 288 96 3 0.423800 2.195352e+01 8.190100e+00 1.017360e+01 # Hw SPCE + 289 97 3 0.423800 5.786826e+00 1.087307e+01 1.086502e+01 # Hw SPCE + 290 97 4 -0.847600 6.324427e+00 1.163284e+01 1.123071e+01 # Ow SPCE + 291 97 3 0.423800 7.142246e+00 1.127568e+01 1.168194e+01 # Hw SPCE + 292 98 3 0.423800 1.143842e+01 2.119323e+01 8.871945e+00 # Hw SPCE + 293 98 4 -0.847600 1.139221e+01 2.127877e+01 7.876682e+00 # Ow SPCE + 294 98 3 0.423800 1.071119e+01 2.064007e+01 7.518516e+00 # Hw SPCE + 295 99 3 0.423800 5.459585e+00 2.383383e+01 5.598160e+00 # Hw SPCE + 296 99 4 -0.847600 5.520099e+00 2.482379e+01 5.470403e+00 # Ow SPCE + 297 99 3 0.423800 5.586370e+00 2.503046e+01 4.494238e+00 # Hw SPCE + 298 100 3 0.423800 2.264458e+00 7.960990e+00 2.770952e+01 # Hw SPCE + 299 100 4 -0.847600 2.090525e+00 8.160734e+00 2.674523e+01 # Ow SPCE + 300 100 3 0.423800 1.495909e+00 7.454707e+00 2.636058e+01 # Hw SPCE + 301 101 3 0.423800 7.095393e+00 1.940453e+01 2.544132e+01 # Hw SPCE + 302 101 4 -0.847600 6.237306e+00 1.963392e+01 2.498190e+01 # Ow SPCE + 303 101 3 0.423800 5.644089e+00 1.882911e+01 2.496253e+01 # Hw SPCE + 304 102 3 0.423800 1.499078e+00 3.404424e+00 2.770580e+01 # Hw SPCE + 305 102 4 -0.847600 2.042815e+00 3.910224e+00 2.703609e+01 # Ow SPCE + 306 102 3 0.423800 1.977853e+00 4.889953e+00 2.722559e+01 # Hw SPCE + 307 103 3 0.423800 5.115452e+00 2.292057e+01 1.247440e+01 # Hw SPCE + 308 103 4 -0.847600 5.773141e+00 2.354653e+01 1.205533e+01 # Ow SPCE + 309 103 3 0.423800 6.680714e+00 2.312664e+01 1.205713e+01 # Hw SPCE + 310 104 3 0.423800 1.944181e+01 2.324594e+01 1.499570e+00 # Hw SPCE + 311 104 4 -0.847600 2.043993e+01 2.318469e+01 1.499022e+00 # Ow SPCE + 312 104 3 0.423800 2.071487e+01 2.222323e+01 1.498289e+00 # Hw SPCE + 313 105 3 0.423800 1.030626e+01 8.957281e+00 6.373247e+00 # Hw SPCE + 314 105 4 -0.847600 9.786014e+00 8.104555e+00 6.326252e+00 # Ow SPCE + 315 105 3 0.423800 8.880748e+00 8.243851e+00 6.727613e+00 # Hw SPCE + 316 106 3 0.423800 1.864859e+01 1.763094e+00 1.290621e+01 # Hw SPCE + 317 106 4 -0.847600 1.923656e+01 2.571543e+00 1.287976e+01 # Ow SPCE + 318 106 3 0.423800 2.017824e+01 2.292795e+00 1.269123e+01 # Hw SPCE + 319 107 3 0.423800 2.129031e+00 5.623946e+00 1.104896e+01 # Hw SPCE + 320 107 4 -0.847600 1.506474e+00 5.035776e+00 1.053275e+01 # Ow SPCE + 321 107 3 0.423800 1.486177e+00 4.126233e+00 1.094786e+01 # Hw SPCE + 322 108 3 0.423800 2.212734e+01 1.355457e+01 3.475366e+00 # Hw SPCE + 323 108 4 -0.847600 2.189942e+01 1.348653e+01 4.446665e+00 # Ow SPCE + 324 108 3 0.423800 2.141982e+01 1.431518e+01 4.735327e+00 # Hw SPCE + 325 109 3 0.423800 2.714041e+01 9.226233e+00 1.942138e+01 # Hw SPCE + 326 109 4 -0.847600 2.695029e+01 1.019981e+01 1.954785e+01 # Ow SPCE + 327 109 3 0.423800 2.770446e+01 1.073827e+01 1.917198e+01 # Hw SPCE + 328 110 3 0.423800 1.145913e+01 1.921390e+01 1.496917e+01 # Hw SPCE + 329 110 4 -0.847600 1.131470e+01 2.018773e+01 1.479367e+01 # Ow SPCE + 330 110 3 0.423800 1.033642e+01 2.036578e+01 1.468754e+01 # Hw SPCE + 331 111 3 0.423800 4.274133e+00 2.306189e+00 2.713568e+01 # Hw SPCE + 332 111 4 -0.847600 4.237246e+00 1.500039e+00 2.654512e+01 # Ow SPCE + 333 111 3 0.423800 3.652688e+00 1.690998e+00 2.575656e+01 # Hw SPCE + 334 112 3 0.423800 2.374365e+01 4.285483e+00 2.548838e+01 # Hw SPCE + 335 112 4 -0.847600 2.363447e+01 3.410101e+00 2.501744e+01 # Ow SPCE + 336 112 3 0.423800 2.453520e+01 3.022399e+00 2.482154e+01 # Hw SPCE + 337 113 3 0.423800 2.147633e+01 2.400029e+01 6.981339e+00 # Hw SPCE + 338 113 4 -0.847600 2.070482e+01 2.460854e+01 6.794782e+00 # Ow SPCE + 339 113 3 0.423800 2.100757e+01 2.555958e+01 6.856910e+00 # Hw SPCE + 340 114 3 0.423800 2.475952e+01 2.542350e+01 2.655084e+01 # Hw SPCE + 341 114 4 -0.847600 2.490466e+01 2.505812e+01 2.563137e+01 # Ow SPCE + 342 114 3 0.423800 2.404295e+01 2.469469e+01 2.527727e+01 # Hw SPCE + 343 115 3 0.423800 2.160335e+01 2.393199e+01 1.009846e+01 # Hw SPCE + 344 115 4 -0.847600 2.078275e+01 2.449002e+01 1.022179e+01 # Ow SPCE + 345 115 3 0.423800 2.003750e+01 2.411294e+01 9.671860e+00 # Hw SPCE + 346 116 3 0.423800 1.865506e+01 2.117845e+01 1.756982e+01 # Hw SPCE + 347 116 4 -0.847600 1.807847e+01 2.183151e+01 1.707884e+01 # Ow SPCE + 348 116 3 0.423800 1.837699e+01 2.276271e+01 1.728801e+01 # Hw SPCE + 349 117 3 0.423800 2.061645e+00 6.549304e+00 1.814025e+01 # Hw SPCE + 350 117 4 -0.847600 1.475173e+00 6.253958e+00 1.889446e+01 # Ow SPCE + 351 117 3 0.423800 1.571483e+00 5.267187e+00 1.902487e+01 # Hw SPCE + 352 118 3 0.423800 1.643494e+01 1.386321e+01 1.572937e+01 # Hw SPCE + 353 118 4 -0.847600 1.600041e+01 1.308756e+01 1.527161e+01 # Ow SPCE + 354 118 3 0.423800 1.500654e+01 1.318983e+01 1.531362e+01 # Hw SPCE + 355 119 3 0.423800 1.886154e+01 1.359977e+01 1.388201e+01 # Hw SPCE + 356 119 4 -0.847600 1.985547e+01 1.361236e+01 1.399129e+01 # Ow SPCE + 357 119 3 0.423800 2.020182e+01 1.267429e+01 1.399938e+01 # Hw SPCE + 358 120 3 0.423800 2.640146e+01 4.249195e+00 1.244508e+01 # Hw SPCE + 359 120 4 -0.847600 2.716791e+01 3.811578e+00 1.291523e+01 # Ow SPCE + 360 120 3 0.423800 2.770629e+01 3.285544e+00 1.225687e+01 # Hw SPCE + 361 121 3 0.423800 1.496812e+00 1.814687e+00 1.782668e+01 # Hw SPCE + 362 121 4 -0.847600 1.815569e+00 1.693357e+00 1.876672e+01 # Ow SPCE + 363 121 3 0.423800 1.625725e+00 2.522412e+00 1.929267e+01 # Hw SPCE + 364 122 3 0.423800 2.680988e+01 1.869516e+01 7.392907e+00 # Hw SPCE + 365 122 4 -0.847600 2.739577e+01 1.944655e+01 7.696489e+00 # Ow SPCE + 366 122 3 0.423800 2.770460e+01 1.997015e+01 6.902469e+00 # Hw SPCE + 367 123 3 0.423800 6.657051e+00 1.748551e+01 3.900748e+00 # Hw SPCE + 368 123 4 -0.847600 7.611764e+00 1.740782e+01 4.187954e+00 # Ow SPCE + 369 123 3 0.423800 7.719232e+00 1.780597e+01 5.098957e+00 # Hw SPCE + 370 124 3 0.423800 1.712795e+01 2.364901e+01 2.137752e+01 # Hw SPCE + 371 124 4 -0.847600 1.651270e+01 2.309626e+01 2.193960e+01 # Ow SPCE + 372 124 3 0.423800 1.702767e+01 2.235135e+01 2.236376e+01 # Hw SPCE + 373 125 3 0.423800 2.465960e+01 9.111990e+00 1.552092e+00 # Hw SPCE + 374 125 4 -0.847600 2.368347e+01 9.328956e+00 1.542385e+00 # Ow SPCE + 375 125 3 0.423800 2.356287e+01 1.032079e+01 1.500818e+00 # Hw SPCE + 376 126 3 0.423800 1.541633e+01 2.513037e+01 2.342964e+01 # Hw SPCE + 377 126 4 -0.847600 1.496712e+01 2.563480e+01 2.416705e+01 # Ow SPCE + 378 126 3 0.423800 1.407826e+01 2.521990e+01 2.436140e+01 # Hw SPCE + 379 127 3 0.423800 2.198927e+01 2.296037e+01 1.264179e+01 # Hw SPCE + 380 127 4 -0.847600 2.135361e+01 2.220618e+01 1.280650e+01 # Ow SPCE + 381 127 3 0.423800 2.142262e+01 2.154325e+01 1.206101e+01 # Hw SPCE + 382 128 3 0.423800 5.808379e+00 1.479597e+00 1.729754e+01 # Hw SPCE + 383 128 4 -0.847600 5.311955e+00 2.244876e+00 1.770731e+01 # Ow SPCE + 384 128 3 0.423800 4.490271e+00 1.903163e+00 1.816345e+01 # Hw SPCE + 385 129 3 0.423800 4.410407e+00 9.586919e+00 1.489573e+01 # Hw SPCE + 386 129 4 -0.847600 4.330005e+00 8.939478e+00 1.413786e+01 # Ow SPCE + 387 129 3 0.423800 5.226071e+00 8.810656e+00 1.371305e+01 # Hw SPCE + 388 130 3 0.423800 8.119198e+00 3.621439e+00 1.317752e+01 # Hw SPCE + 389 130 4 -0.847600 8.167483e+00 2.832111e+00 1.256545e+01 # Ow SPCE + 390 130 3 0.423800 7.317941e+00 2.308454e+00 1.262918e+01 # Hw SPCE + 391 131 3 0.423800 1.605369e+01 6.725170e+00 2.132125e+01 # Hw SPCE + 392 131 4 -0.847600 1.601692e+01 7.081523e+00 2.225488e+01 # Ow SPCE + 393 131 3 0.423800 1.677297e+01 7.720235e+00 2.239787e+01 # Hw SPCE + 394 132 3 0.423800 2.587812e+01 2.770384e+01 6.790070e+00 # Hw SPCE + 395 132 4 -0.847600 2.495789e+01 2.770683e+01 6.398688e+00 # Ow SPCE + 396 132 3 0.423800 2.428216e+01 2.770348e+01 7.135826e+00 # Hw SPCE + 397 133 3 0.423800 3.242275e+00 2.430164e+01 3.033818e+00 # Hw SPCE + 398 133 4 -0.847600 3.385077e+00 2.527679e+01 2.864385e+00 # Ow SPCE + 399 133 3 0.423800 4.115488e+00 2.539474e+01 2.191640e+00 # Hw SPCE + 400 134 3 0.423800 1.391872e+01 1.853150e+01 1.389147e+01 # Hw SPCE + 401 134 4 -0.847600 1.350915e+01 1.835748e+01 1.299594e+01 # Ow SPCE + 402 134 3 0.423800 1.277157e+01 1.768895e+01 1.309098e+01 # Hw SPCE + 403 135 3 0.423800 1.883256e+00 1.950562e+01 6.097187e+00 # Hw SPCE + 404 135 4 -0.847600 2.108048e+00 1.853214e+01 6.139671e+00 # Ow SPCE + 405 135 3 0.423800 1.664096e+00 1.805416e+01 5.381755e+00 # Hw SPCE + 406 136 3 0.423800 1.772073e+01 1.904358e+01 6.835467e+00 # Hw SPCE + 407 136 4 -0.847600 1.848360e+01 1.839868e+01 6.881595e+00 # Ow SPCE + 408 136 3 0.423800 1.815821e+01 1.748070e+01 6.654789e+00 # Hw SPCE + 409 137 3 0.423800 1.668142e+01 1.746799e+01 2.770692e+01 # Hw SPCE + 410 137 4 -0.847600 1.690378e+01 1.739205e+01 2.673492e+01 # Ow SPCE + 411 137 3 0.423800 1.607886e+01 1.715496e+01 2.622181e+01 # Hw SPCE + 412 138 3 0.423800 6.510214e+00 5.350768e+00 8.615350e+00 # Hw SPCE + 413 138 4 -0.847600 6.301757e+00 6.322395e+00 8.727089e+00 # Ow SPCE + 414 138 3 0.423800 7.064288e+00 6.868862e+00 8.380795e+00 # Hw SPCE + 415 139 3 0.423800 1.123604e+01 1.437475e+01 2.577805e+01 # Hw SPCE + 416 139 4 -0.847600 1.147623e+01 1.512029e+01 2.639973e+01 # Ow SPCE + 417 139 3 0.423800 1.064787e+01 1.561484e+01 2.666285e+01 # Hw SPCE + 418 140 3 0.423800 1.018251e+01 2.301900e+01 2.071248e+01 # Hw SPCE + 419 140 4 -0.847600 1.073103e+01 2.232837e+01 2.118382e+01 # Ow SPCE + 420 140 3 0.423800 1.020516e+01 2.148141e+01 2.126196e+01 # Hw SPCE + 421 141 3 0.423800 5.434060e+00 2.503928e+00 1.462372e+01 # Hw SPCE + 422 141 4 -0.847600 6.195222e+00 3.000981e+00 1.504034e+01 # Ow SPCE + 423 141 3 0.423800 5.916457e+00 3.943344e+00 1.522539e+01 # Hw SPCE + 424 142 3 0.423800 2.396703e+01 1.692595e+01 1.892427e+01 # Hw SPCE + 425 142 4 -0.847600 2.487591e+01 1.651107e+01 1.888188e+01 # Ow SPCE + 426 142 3 0.423800 2.528573e+01 1.669671e+01 1.798880e+01 # Hw SPCE + 427 143 3 0.423800 1.739023e+01 2.770535e+01 2.756282e+01 # Hw SPCE + 428 143 4 -0.847600 1.824208e+01 2.718404e+01 2.761360e+01 # Ow SPCE + 429 143 3 0.423800 1.803237e+01 2.621369e+01 2.773377e+01 # Hw SPCE + 430 144 3 0.423800 1.002054e+01 2.068443e+00 2.080057e+01 # Hw SPCE + 431 144 4 -0.847600 1.037196e+01 1.499358e+00 2.005717e+01 # Ow SPCE + 432 144 3 0.423800 1.137167e+01 1.499631e+00 2.008118e+01 # Hw SPCE + 433 145 3 0.423800 5.474756e+00 6.597903e+00 2.212606e+01 # Hw SPCE + 434 145 4 -0.847600 6.060290e+00 6.454103e+00 2.292385e+01 # Ow SPCE + 435 145 3 0.423800 5.607820e+00 5.830491e+00 2.356133e+01 # Hw SPCE + 436 146 3 0.423800 9.616079e+00 2.770577e+01 1.272301e+01 # Hw SPCE + 437 146 4 -0.847600 1.005389e+01 2.758118e+01 1.183261e+01 # Ow SPCE + 438 146 3 0.423800 1.104172e+01 2.770510e+01 1.192665e+01 # Hw SPCE + 439 147 3 0.423800 7.556371e+00 8.751423e+00 1.067315e+01 # Hw SPCE + 440 147 4 -0.847600 6.677067e+00 8.276682e+00 1.071118e+01 # Ow SPCE + 441 147 3 0.423800 6.692592e+00 7.605177e+00 1.145202e+01 # Hw SPCE + 442 148 3 0.423800 1.048902e+01 9.944021e+00 1.499856e+00 # Hw SPCE + 443 148 4 -0.847600 1.107788e+01 9.135791e+00 1.497048e+00 # Ow SPCE + 444 148 3 0.423800 1.051214e+01 8.311210e+00 1.495741e+00 # Hw SPCE + 445 149 3 0.423800 2.225224e+01 1.699334e+01 1.388572e+01 # Hw SPCE + 446 149 4 -0.847600 2.247601e+01 1.610910e+01 1.347578e+01 # Ow SPCE + 447 149 3 0.423800 2.332230e+01 1.619040e+01 1.294930e+01 # Hw SPCE + 448 150 3 0.423800 1.629210e+01 2.770573e+01 8.693261e+00 # Hw SPCE + 449 150 4 -0.847600 1.560110e+01 2.709673e+01 8.303856e+00 # Ow SPCE + 450 150 3 0.423800 1.580582e+01 2.615242e+01 8.561464e+00 # Hw SPCE + 451 151 3 0.423800 2.423080e+01 1.358698e+01 1.852585e+01 # Hw SPCE + 452 151 4 -0.847600 2.350560e+01 1.411138e+01 1.897204e+01 # Ow SPCE + 453 151 3 0.423800 2.267128e+01 1.406026e+01 1.842312e+01 # Hw SPCE + 454 152 3 0.423800 2.207406e+01 4.433705e+00 2.065867e+01 # Hw SPCE + 455 152 4 -0.847600 2.276959e+01 3.724600e+00 2.054287e+01 # Ow SPCE + 456 152 3 0.423800 2.359197e+01 4.126886e+00 2.014056e+01 # Hw SPCE + 457 153 3 0.423800 2.395702e+01 6.074153e+00 1.705293e+01 # Hw SPCE + 458 153 4 -0.847600 2.358426e+01 5.192507e+00 1.734233e+01 # Ow SPCE + 459 153 3 0.423800 2.433467e+01 4.558278e+00 1.752846e+01 # Hw SPCE + 460 154 3 0.423800 1.155557e+01 2.807646e+00 1.636444e+01 # Hw SPCE + 461 154 4 -0.847600 1.146110e+01 1.812305e+00 1.638374e+01 # Ow SPCE + 462 154 3 0.423800 1.138045e+01 1.503463e+00 1.733143e+01 # Hw SPCE + 463 155 3 0.423800 4.135912e+00 2.531900e+01 1.818837e+01 # Hw SPCE + 464 155 4 -0.847600 3.741676e+00 2.521934e+01 1.910196e+01 # Ow SPCE + 465 155 3 0.423800 4.213837e+00 2.448559e+01 1.959050e+01 # Hw SPCE + 466 156 3 0.423800 1.499258e+00 1.198985e+01 2.578979e+01 # Hw SPCE + 467 156 4 -0.847600 2.062832e+00 1.128002e+01 2.536727e+01 # Ow SPCE + 468 156 3 0.423800 1.537875e+00 1.043123e+01 2.530414e+01 # Hw SPCE + 469 157 3 0.423800 6.788951e+00 2.673524e+01 2.771333e+01 # Hw SPCE + 470 157 4 -0.847600 6.736475e+00 2.763375e+01 2.727752e+01 # Ow SPCE + 471 157 3 0.423800 5.853127e+00 2.773069e+01 2.681893e+01 # Hw SPCE + 472 158 3 0.423800 4.823100e+00 1.350860e+01 6.387855e+00 # Hw SPCE + 473 158 4 -0.847600 4.369663e+00 1.263897e+01 6.583146e+00 # Ow SPCE + 474 158 3 0.423800 4.571234e+00 1.236153e+01 7.522508e+00 # Hw SPCE + 475 159 3 0.423800 2.652742e+01 8.643512e+00 1.208390e+01 # Hw SPCE + 476 159 4 -0.847600 2.750728e+01 8.461636e+00 1.216639e+01 # Ow SPCE + 477 159 3 0.423800 2.770441e+01 7.539751e+00 1.183282e+01 # Hw SPCE + 478 160 3 0.423800 1.562548e+01 7.342341e+00 1.368183e+01 # Hw SPCE + 479 160 4 -0.847600 1.644892e+01 7.724632e+00 1.326255e+01 # Ow SPCE + 480 160 3 0.423800 1.657765e+01 7.324209e+00 1.235531e+01 # Hw SPCE + 481 161 3 0.423800 1.832876e+01 3.664241e+00 1.523424e+01 # Hw SPCE + 482 161 4 -0.847600 1.757342e+01 3.228540e+00 1.572376e+01 # Ow SPCE + 483 161 3 0.423800 1.723220e+01 2.455027e+00 1.518968e+01 # Hw SPCE + 484 162 3 0.423800 1.163979e+01 2.726442e+01 5.638394e+00 # Hw SPCE + 485 162 4 -0.847600 1.203117e+01 2.637033e+01 5.856191e+00 # Ow SPCE + 486 162 3 0.423800 1.265936e+01 2.646077e+01 6.628973e+00 # Hw SPCE + 487 163 3 0.423800 3.712836e+00 1.795600e+01 2.104265e+01 # Hw SPCE + 488 163 4 -0.847600 3.729339e+00 1.776723e+01 2.202453e+01 # Ow SPCE + 489 163 3 0.423800 2.793088e+00 1.774262e+01 2.237500e+01 # Hw SPCE + 490 164 3 0.423800 1.215495e+01 1.501989e+00 9.660698e+00 # Hw SPCE + 491 164 4 -0.847600 1.306022e+01 1.479590e+00 9.236442e+00 # Ow SPCE + 492 164 3 0.423800 1.376231e+01 1.546750e+00 9.945354e+00 # Hw SPCE + 493 165 3 0.423800 1.382068e+01 2.627313e+01 1.928618e+01 # Hw SPCE + 494 165 4 -0.847600 1.369118e+01 2.724037e+01 1.950452e+01 # Ow SPCE + 495 165 3 0.423800 1.457266e+01 2.771119e+01 1.946813e+01 # Hw SPCE + 496 166 3 0.423800 1.936570e+01 1.864386e+01 2.521456e+01 # Hw SPCE + 497 166 4 -0.847600 2.026756e+01 1.895792e+01 2.551126e+01 # Ow SPCE + 498 166 3 0.423800 2.023242e+01 1.993867e+01 2.570334e+01 # Hw SPCE + 499 167 3 0.423800 1.169885e+01 2.172581e+00 2.751632e+01 # Hw SPCE + 500 167 4 -0.847600 1.090011e+01 1.573456e+00 2.746098e+01 # Ow SPCE + 501 167 3 0.423800 1.061137e+01 1.490153e+00 2.650720e+01 # Hw SPCE + 502 168 3 0.423800 1.911213e+01 2.158443e+01 1.022983e+01 # Hw SPCE + 503 168 4 -0.847600 1.926141e+01 2.060202e+01 1.011766e+01 # Ow SPCE + 504 168 3 0.423800 1.928483e+01 2.016370e+01 1.101617e+01 # Hw SPCE + 505 169 3 0.423800 1.629210e+00 1.550801e+01 1.095958e+01 # Hw SPCE + 506 169 4 -0.847600 1.493041e+00 1.635194e+01 1.147846e+01 # Ow SPCE + 507 169 3 0.423800 2.168035e+00 1.703190e+01 1.119204e+01 # Hw SPCE + 508 170 3 0.423800 1.102279e+01 4.173853e+00 1.869068e+01 # Hw SPCE + 509 170 4 -0.847600 1.109242e+01 4.502781e+00 1.963246e+01 # Ow SPCE + 510 170 3 0.423800 1.122948e+01 5.493340e+00 1.963029e+01 # Hw SPCE + 511 171 3 0.423800 8.600246e+00 2.481675e+01 1.935975e+01 # Hw SPCE + 512 171 4 -0.847600 8.003650e+00 2.561907e+01 1.934092e+01 # Ow SPCE + 513 171 3 0.423800 7.051209e+00 2.532446e+01 1.926308e+01 # Hw SPCE + 514 172 3 0.423800 7.877713e+00 1.822785e+01 1.950056e+01 # Hw SPCE + 515 172 4 -0.847600 6.923133e+00 1.800620e+01 1.930144e+01 # Ow SPCE + 516 172 3 0.423800 6.847202e+00 1.767403e+01 1.836128e+01 # Hw SPCE + 517 173 3 0.423800 2.715931e+01 1.264839e+01 1.341405e+01 # Hw SPCE + 518 173 4 -0.847600 2.734880e+01 1.167554e+01 1.328112e+01 # Ow SPCE + 519 173 3 0.423800 2.648989e+01 1.118630e+01 1.312977e+01 # Hw SPCE + 520 174 3 0.423800 1.482572e+01 8.447714e+00 1.055548e+01 # Hw SPCE + 521 174 4 -0.847600 1.531227e+01 7.960482e+00 9.830314e+00 # Ow SPCE + 522 174 3 0.423800 1.465158e+01 7.577469e+00 9.184724e+00 # Hw SPCE + 523 175 3 0.423800 6.350160e+00 1.781671e+00 2.021119e+01 # Hw SPCE + 524 175 4 -0.847600 5.554766e+00 1.449580e+00 2.071820e+01 # Ow SPCE + 525 175 3 0.423800 4.990992e+00 2.226040e+00 2.099974e+01 # Hw SPCE + 526 176 3 0.423800 2.360588e+01 2.124534e+01 1.405838e+01 # Hw SPCE + 527 176 4 -0.847600 2.343533e+01 2.028764e+01 1.429017e+01 # Ow SPCE + 528 176 3 0.423800 2.247062e+01 2.007410e+01 1.413605e+01 # Hw SPCE + 529 177 3 0.423800 2.476177e+01 9.495265e+00 2.278084e+01 # Hw SPCE + 530 177 4 -0.847600 2.490594e+01 9.103976e+00 2.187194e+01 # Ow SPCE + 531 177 3 0.423800 2.584301e+01 8.762128e+00 2.180102e+01 # Hw SPCE + 532 178 3 0.423800 2.124801e+01 1.026202e+01 1.226324e+01 # Hw SPCE + 533 178 4 -0.847600 2.175978e+01 1.061858e+01 1.304488e+01 # Ow SPCE + 534 178 3 0.423800 2.143033e+01 1.018622e+01 1.388424e+01 # Hw SPCE + 535 179 3 0.423800 2.770461e+01 2.265303e+01 1.897481e+01 # Hw SPCE + 536 179 4 -0.847600 2.771622e+01 2.165817e+01 1.887413e+01 # Ow SPCE + 537 179 3 0.423800 2.699937e+01 2.137944e+01 1.823504e+01 # Hw SPCE + 538 180 3 0.423800 1.752412e+00 1.211560e+01 2.448378e+00 # Hw SPCE + 539 180 4 -0.847600 2.192876e+00 1.124630e+01 2.672646e+00 # Ow SPCE + 540 180 3 0.423800 1.502189e+00 1.052455e+01 2.717762e+00 # Hw SPCE + 541 181 3 0.423800 2.066796e+01 9.492666e+00 1.990433e+01 # Hw SPCE + 542 181 4 -0.847600 1.987345e+01 1.007578e+01 1.973481e+01 # Ow SPCE + 543 181 3 0.423800 1.945518e+01 1.033091e+01 2.060657e+01 # Hw SPCE + 544 182 3 0.423800 7.526139e+00 1.353553e+01 1.705107e+01 # Hw SPCE + 545 182 4 -0.847600 7.411920e+00 1.404119e+01 1.619593e+01 # Ow SPCE + 546 182 3 0.423800 7.814322e+00 1.495172e+01 1.629081e+01 # Hw SPCE + 547 183 3 0.423800 5.320834e+00 2.489018e+01 2.627333e+01 # Hw SPCE + 548 183 4 -0.847600 4.512268e+00 2.533586e+01 2.665751e+01 # Ow SPCE + 549 183 3 0.423800 3.697906e+00 2.503309e+01 2.616239e+01 # Hw SPCE + 550 184 3 0.423800 2.555518e+01 2.496555e+01 1.499745e+00 # Hw SPCE + 551 184 4 -0.847600 2.580931e+01 2.400607e+01 1.621410e+00 # Ow SPCE + 552 184 3 0.423800 2.498688e+01 2.343773e+01 1.596790e+00 # Hw SPCE + 553 185 3 0.423800 4.020936e+00 8.516876e+00 4.121746e+00 # Hw SPCE + 554 185 4 -0.847600 3.704058e+00 9.464727e+00 4.155894e+00 # Ow SPCE + 555 185 3 0.423800 4.491592e+00 1.008053e+01 4.131828e+00 # Hw SPCE + 556 186 3 0.423800 1.411577e+01 1.487915e+00 1.833251e+01 # Hw SPCE + 557 186 4 -0.847600 1.431943e+01 2.462252e+00 1.842837e+01 # Ow SPCE + 558 186 3 0.423800 1.347701e+01 2.990756e+00 1.832346e+01 # Hw SPCE + 559 187 3 0.423800 3.071401e+00 9.241825e+00 1.816214e+01 # Hw SPCE + 560 187 4 -0.847600 3.300129e+00 8.845865e+00 1.727282e+01 # Ow SPCE + 561 187 3 0.423800 2.487941e+00 8.845219e+00 1.668942e+01 # Hw SPCE + 562 188 3 0.423800 1.493937e+00 2.324979e+01 2.604098e+01 # Hw SPCE + 563 188 4 -0.847600 1.544148e+00 2.354108e+01 2.699630e+01 # Ow SPCE + 564 188 3 0.423800 1.493692e+00 2.273962e+01 2.759223e+01 # Hw SPCE + 565 189 3 0.423800 2.402950e+01 7.504644e+00 1.192867e+01 # Hw SPCE + 566 189 4 -0.847600 2.413727e+01 6.710938e+00 1.132999e+01 # Ow SPCE + 567 189 3 0.423800 2.508872e+01 6.403895e+00 1.135159e+01 # Hw SPCE + 568 190 3 0.423800 8.545646e+00 3.156383e+00 9.861230e+00 # Hw SPCE + 569 190 4 -0.847600 9.351073e+00 2.611171e+00 1.009367e+01 # Ow SPCE + 570 190 3 0.423800 1.011166e+01 3.219788e+00 1.031971e+01 # Hw SPCE + 571 191 3 0.423800 1.499645e+00 2.645193e+01 1.809386e+01 # Hw SPCE + 572 191 4 -0.847600 1.941124e+00 2.732475e+01 1.788582e+01 # Ow SPCE + 573 191 3 0.423800 2.336801e+00 2.770568e+01 1.872148e+01 # Hw SPCE + 574 192 3 0.423800 1.070874e+01 1.082303e+01 1.860632e+01 # Hw SPCE + 575 192 4 -0.847600 1.140325e+01 1.152845e+01 1.874785e+01 # Ow SPCE + 576 192 3 0.423800 1.230497e+01 1.110065e+01 1.881024e+01 # Hw SPCE + 577 193 3 0.423800 1.341290e+01 1.795517e+01 2.013626e+01 # Hw SPCE + 578 193 4 -0.847600 1.307952e+01 1.701238e+01 2.013931e+01 # Ow SPCE + 579 193 3 0.423800 1.309126e+01 1.665771e+01 2.107422e+01 # Hw SPCE + 580 194 3 0.423800 4.807785e+00 9.950438e+00 2.034530e+01 # Hw SPCE + 581 194 4 -0.847600 5.368387e+00 9.471078e+00 2.102053e+01 # Ow SPCE + 582 194 3 0.423800 6.091190e+00 8.962304e+00 2.055288e+01 # Hw SPCE + 583 195 3 0.423800 4.581666e+00 2.365989e+01 9.523923e+00 # Hw SPCE + 584 195 4 -0.847600 4.035775e+00 2.430184e+01 1.006236e+01 # Ow SPCE + 585 195 3 0.423800 3.083838e+00 2.426678e+01 9.758078e+00 # Hw SPCE + 586 196 3 0.423800 1.996454e+01 2.420988e+01 1.557209e+01 # Hw SPCE + 587 196 4 -0.847600 2.025724e+01 2.408856e+01 1.462361e+01 # Ow SPCE + 588 196 3 0.423800 2.091875e+01 2.479965e+01 1.438538e+01 # Hw SPCE + 589 197 3 0.423800 3.003372e+00 1.643713e+01 1.704879e+01 # Hw SPCE + 590 197 4 -0.847600 2.577322e+00 1.558424e+01 1.735056e+01 # Ow SPCE + 591 197 3 0.423800 1.583596e+00 1.569510e+01 1.736541e+01 # Hw SPCE + 592 198 3 0.423800 8.582183e+00 2.224273e+01 1.027837e+01 # Hw SPCE + 593 198 4 -0.847600 8.111157e+00 2.153125e+01 9.756897e+00 # Ow SPCE + 594 198 3 0.423800 8.773035e+00 2.084403e+01 9.457494e+00 # Hw SPCE + 595 199 3 0.423800 9.338267e+00 1.004752e+01 1.312920e+01 # Hw SPCE + 596 199 4 -0.847600 8.797900e+00 9.207329e+00 1.308363e+01 # Ow SPCE + 597 199 3 0.423800 7.916778e+00 9.352906e+00 1.353355e+01 # Hw SPCE + 598 200 3 0.423800 2.194779e+01 1.497417e+00 2.193037e+01 # Hw SPCE + 599 200 4 -0.847600 2.223387e+01 1.544058e+00 2.288744e+01 # Ow SPCE + 600 200 3 0.423800 2.323149e+01 1.499141e+00 2.293970e+01 # Hw SPCE + 601 201 3 0.423800 6.357928e+00 6.228009e+00 5.806627e+00 # Hw SPCE + 602 201 4 -0.847600 6.047437e+00 7.177648e+00 5.848827e+00 # Ow SPCE + 603 201 3 0.423800 6.617086e+00 7.741462e+00 5.250820e+00 # Hw SPCE + 604 202 3 0.423800 1.593029e+01 1.490608e+01 1.881990e+00 # Hw SPCE + 605 202 4 -0.847600 1.499401e+01 1.512044e+01 1.603720e+00 # Ow SPCE + 606 202 3 0.423800 1.441686e+01 1.431299e+01 1.725826e+00 # Hw SPCE + 607 203 3 0.423800 2.152874e+01 7.128493e+00 1.779600e+01 # Hw SPCE + 608 203 4 -0.847600 2.073272e+01 6.564299e+00 1.757685e+01 # Ow SPCE + 609 203 3 0.423800 2.013013e+01 6.514325e+00 1.837333e+01 # Hw SPCE + 610 204 3 0.423800 1.867722e+01 1.499875e+00 1.016897e+01 # Hw SPCE + 611 204 4 -0.847600 1.948249e+01 2.057017e+00 9.966143e+00 # Ow SPCE + 612 204 3 0.423800 2.030669e+01 1.498892e+00 1.006198e+01 # Hw SPCE + 613 205 3 0.423800 2.316922e+01 1.367353e+01 6.885430e+00 # Hw SPCE + 614 205 4 -0.847600 2.260716e+01 1.285895e+01 7.028829e+00 # Ow SPCE + 615 205 3 0.423800 2.319607e+01 1.205233e+01 7.079429e+00 # Hw SPCE + 616 206 3 0.423800 1.724802e+01 2.452784e+01 1.522277e+01 # Hw SPCE + 617 206 4 -0.847600 1.695269e+01 2.377047e+01 1.464039e+01 # Ow SPCE + 618 206 3 0.423800 1.764677e+01 2.305066e+01 1.465218e+01 # Hw SPCE + 619 207 3 0.423800 2.411268e+01 4.355955e+00 1.477705e+01 # Hw SPCE + 620 207 4 -0.847600 2.346927e+01 5.080864e+00 1.453102e+01 # Ow SPCE + 621 207 3 0.423800 2.313742e+01 4.930075e+00 1.359982e+01 # Hw SPCE + 622 208 3 0.423800 4.859519e+00 1.208976e+01 2.528815e+01 # Hw SPCE + 623 208 4 -0.847600 4.768744e+00 1.304357e+01 2.557451e+01 # Ow SPCE + 624 208 3 0.423800 3.968809e+00 1.313902e+01 2.616696e+01 # Hw SPCE + 625 209 3 0.423800 1.172825e+01 2.418010e+01 2.667409e+01 # Hw SPCE + 626 209 4 -0.847600 1.132283e+01 2.504940e+01 2.639133e+01 # Ow SPCE + 627 209 3 0.423800 1.154586e+01 2.522495e+01 2.543246e+01 # Hw SPCE + 628 210 3 0.423800 1.341122e+01 2.770513e+01 2.770549e+01 # Hw SPCE + 629 210 4 -0.847600 1.245960e+01 2.739875e+01 2.768238e+01 # Ow SPCE + 630 210 3 0.423800 1.204322e+01 2.766958e+01 2.681446e+01 # Hw SPCE + 631 211 3 0.423800 1.145241e+01 8.302455e+00 1.779676e+01 # Hw SPCE + 632 211 4 -0.847600 1.125478e+01 7.343016e+00 1.759571e+01 # Ow SPCE + 633 211 3 0.423800 1.210977e+01 6.857989e+00 1.741202e+01 # Hw SPCE + 634 212 3 0.423800 1.287925e+01 1.297774e+01 2.765935e+01 # Hw SPCE + 635 212 4 -0.847600 1.230585e+01 1.215930e+01 2.769630e+01 # Ow SPCE + 636 212 3 0.423800 1.134277e+01 1.242772e+01 2.771694e+01 # Hw SPCE + 637 213 3 0.423800 1.353894e+01 1.030050e+01 8.981176e+00 # Hw SPCE + 638 213 4 -0.847600 1.336051e+01 1.105021e+01 8.343917e+00 # Ow SPCE + 639 213 3 0.423800 1.239877e+01 1.103348e+01 8.070479e+00 # Hw SPCE + 640 214 3 0.423800 1.379608e+01 1.561648e+01 4.066143e+00 # Hw SPCE + 641 214 4 -0.847600 1.425485e+01 1.494262e+01 4.645317e+00 # Ow SPCE + 642 214 3 0.423800 1.500310e+01 1.538348e+01 5.141055e+00 # Hw SPCE + 643 215 3 0.423800 1.743267e+01 1.012996e+01 9.164174e+00 # Hw SPCE + 644 215 4 -0.847600 1.689952e+01 1.019764e+01 1.000748e+01 # Ow SPCE + 645 215 3 0.423800 1.610254e+01 1.078129e+01 9.852029e+00 # Hw SPCE + 646 216 3 0.423800 2.449752e+01 2.458848e+01 1.376845e+01 # Hw SPCE + 647 216 4 -0.847600 2.528553e+01 2.495639e+01 1.426210e+01 # Ow SPCE + 648 216 3 0.423800 2.534289e+01 2.452894e+01 1.516431e+01 # Hw SPCE + 649 217 3 0.423800 1.840558e+01 3.099439e+00 2.461987e+01 # Hw SPCE + 650 217 4 -0.847600 1.888190e+01 2.411523e+00 2.516749e+01 # Ow SPCE + 651 217 3 0.423800 1.859125e+01 1.498335e+00 2.488182e+01 # Hw SPCE + 652 218 3 0.423800 2.313333e+01 2.163166e+01 2.205864e+01 # Hw SPCE + 653 218 4 -0.847600 2.219594e+01 2.130927e+01 2.219041e+01 # Ow SPCE + 654 218 3 0.423800 2.158294e+01 2.209428e+01 2.227978e+01 # Hw SPCE + 655 219 3 0.423800 1.229832e+01 6.995752e+00 1.467363e+01 # Hw SPCE + 656 219 4 -0.847600 1.306529e+01 6.454869e+00 1.432837e+01 # Ow SPCE + 657 219 3 0.423800 1.350970e+01 5.982810e+00 1.508972e+01 # Hw SPCE + 658 220 3 0.423800 1.406803e+01 2.706445e+00 1.555541e+00 # Hw SPCE + 659 220 4 -0.847600 1.438807e+01 3.652120e+00 1.498299e+00 # Ow SPCE + 660 220 3 0.423800 1.360529e+01 4.272470e+00 1.547574e+00 # Hw SPCE + 661 221 3 0.423800 1.346094e+01 1.584618e+01 1.101422e+01 # Hw SPCE + 662 221 4 -0.847600 1.417123e+01 1.647755e+01 1.070299e+01 # Ow SPCE + 663 221 3 0.423800 1.506879e+01 1.605138e+01 1.081592e+01 # Hw SPCE + 664 222 3 0.423800 6.578749e+00 2.770463e+01 1.007095e+01 # Hw SPCE + 665 222 4 -0.847600 5.578913e+00 2.770497e+01 1.008903e+01 # Ow SPCE + 666 222 3 0.423800 5.245578e+00 2.676237e+01 1.010848e+01 # Hw SPCE + 667 223 3 0.423800 2.346524e+01 1.409205e+01 2.397168e+01 # Hw SPCE + 668 223 4 -0.847600 2.273539e+01 1.354071e+01 2.356753e+01 # Ow SPCE + 669 223 3 0.423800 2.313609e+01 1.282550e+01 2.299489e+01 # Hw SPCE + 670 224 3 0.423800 9.455855e+00 2.740531e+01 2.770496e+01 # Hw SPCE + 671 224 4 -0.847600 9.370999e+00 2.698567e+01 2.680125e+01 # Ow SPCE + 672 224 3 0.423800 9.379656e+00 2.769893e+01 2.610040e+01 # Hw SPCE + 673 225 3 0.423800 1.739219e+01 2.174463e+01 3.666550e+00 # Hw SPCE + 674 225 4 -0.847600 1.786402e+01 2.143690e+01 2.840303e+00 # Ow SPCE + 675 225 3 0.423800 1.877157e+01 2.109408e+01 3.082833e+00 # Hw SPCE + 676 226 3 0.423800 2.616571e+01 6.000875e+00 7.270531e+00 # Hw SPCE + 677 226 4 -0.847600 2.581662e+01 5.090419e+00 7.048703e+00 # Ow SPCE + 678 226 3 0.423800 2.500801e+01 5.176179e+00 6.466644e+00 # Hw SPCE + 679 227 3 0.423800 2.463564e+01 9.703885e+00 1.039308e+01 # Hw SPCE + 680 227 4 -0.847600 2.383949e+01 1.018568e+01 1.002699e+01 # Ow SPCE + 681 227 3 0.423800 2.359408e+01 1.093724e+01 1.063930e+01 # Hw SPCE + 682 228 3 0.423800 2.177044e+01 1.687390e+01 2.431866e+01 # Hw SPCE + 683 228 4 -0.847600 2.133664e+01 1.645723e+01 2.511753e+01 # Ow SPCE + 684 228 3 0.423800 2.195993e+01 1.578987e+01 2.552514e+01 # Hw SPCE + 685 229 3 0.423800 7.726348e+00 3.560323e+00 2.179374e+01 # Hw SPCE + 686 229 4 -0.847600 6.861868e+00 4.061639e+00 2.183058e+01 # Ow SPCE + 687 229 3 0.423800 6.645800e+00 4.420653e+00 2.092261e+01 # Hw SPCE + 688 230 3 0.423800 1.884037e+01 1.499599e+00 6.252581e+00 # Hw SPCE + 689 230 4 -0.847600 1.965401e+01 2.001138e+00 6.546590e+00 # Ow SPCE + 690 230 3 0.423800 2.008019e+01 1.523837e+00 7.315065e+00 # Hw SPCE + 691 231 3 0.423800 6.715548e+00 3.056808e+00 2.444232e+01 # Hw SPCE + 692 231 4 -0.847600 6.554914e+00 2.209760e+00 2.393566e+01 # Ow SPCE + 693 231 3 0.423800 6.112326e+00 1.542117e+00 2.453429e+01 # Hw SPCE + 694 232 3 0.423800 1.335233e+01 8.701900e+00 2.332491e+01 # Hw SPCE + 695 232 4 -0.847600 1.284324e+01 9.531573e+00 2.355397e+01 # Ow SPCE + 696 232 3 0.423800 1.192981e+01 9.484446e+00 2.314971e+01 # Hw SPCE + 697 233 3 0.423800 1.642811e+01 5.397368e+00 2.093058e+00 # Hw SPCE + 698 233 4 -0.847600 1.732324e+01 5.692575e+00 1.759011e+00 # Ow SPCE + 699 233 3 0.423800 1.732664e+01 6.686459e+00 1.648636e+00 # Hw SPCE + 700 234 3 0.423800 8.746384e+00 1.194555e+01 1.502826e+01 # Hw SPCE + 701 234 4 -0.847600 8.569902e+00 1.110875e+01 1.554655e+01 # Ow SPCE + 702 234 3 0.423800 9.214489e+00 1.104644e+01 1.630854e+01 # Hw SPCE + 703 235 3 0.423800 2.671040e+01 2.277390e+01 2.443136e+01 # Hw SPCE + 704 235 4 -0.847600 2.634248e+01 2.188483e+01 2.415900e+01 # Ow SPCE + 705 235 3 0.423800 2.540832e+01 2.199999e+01 2.382124e+01 # Hw SPCE + 706 236 3 0.423800 4.314700e+00 6.656624e+00 1.567087e+01 # Hw SPCE + 707 236 4 -0.847600 4.101082e+00 5.989344e+00 1.495735e+01 # Ow SPCE + 708 236 3 0.423800 4.681834e+00 6.155976e+00 1.416051e+01 # Hw SPCE + 709 237 3 0.423800 2.585970e+01 5.937546e+00 1.935423e+01 # Hw SPCE + 710 237 4 -0.847600 2.680819e+01 5.995046e+00 1.966578e+01 # Ow SPCE + 711 237 3 0.423800 2.685723e+01 6.574317e+00 2.047944e+01 # Hw SPCE + 712 238 3 0.423800 2.668626e+01 2.541072e+01 4.880638e+00 # Hw SPCE + 713 238 4 -0.847600 2.764667e+01 2.568864e+01 4.899778e+00 # Ow SPCE + 714 238 3 0.423800 2.770442e+01 2.668068e+01 5.011688e+00 # Hw SPCE + 715 239 3 0.423800 3.799348e+00 1.733873e+01 1.665944e+00 # Hw SPCE + 716 239 4 -0.847600 3.166531e+00 1.811195e+01 1.625052e+00 # Ow SPCE + 717 239 3 0.423800 2.227096e+00 1.777190e+01 1.582255e+00 # Hw SPCE + 718 240 3 0.423800 6.922100e+00 1.626292e+01 2.727964e+01 # Hw SPCE + 719 240 4 -0.847600 6.522882e+00 1.559908e+01 2.664724e+01 # Ow SPCE + 720 240 3 0.423800 6.008322e+00 1.491453e+01 2.716360e+01 # Hw SPCE + 721 241 3 0.423800 3.812524e+00 1.917142e+01 4.078941e+00 # Hw SPCE + 722 241 4 -0.847600 4.650408e+00 1.921783e+01 4.622812e+00 # Ow SPCE + 723 241 3 0.423800 5.291688e+00 1.985376e+01 4.193446e+00 # Hw SPCE + 724 242 3 0.423800 2.392996e+01 9.703159e+00 1.533263e+01 # Hw SPCE + 725 242 4 -0.847600 2.418034e+01 1.027088e+01 1.454841e+01 # Ow SPCE + 726 242 3 0.423800 2.425829e+01 9.697253e+00 1.373301e+01 # Hw SPCE + 727 243 3 0.423800 1.160740e+01 4.289690e+00 2.576183e+01 # Hw SPCE + 728 243 4 -0.847600 1.136615e+01 3.672164e+00 2.501319e+01 # Ow SPCE + 729 243 3 0.423800 1.134214e+01 4.183341e+00 2.415404e+01 # Hw SPCE + 730 244 3 0.423800 3.062271e+00 1.498880e+00 1.558829e+01 # Hw SPCE + 731 244 4 -0.847600 2.459085e+00 1.579929e+00 1.479482e+01 # Ow SPCE + 732 244 3 0.423800 1.507109e+00 1.499362e+00 1.509020e+01 # Hw SPCE + 733 245 3 0.423800 9.267839e+00 2.530564e+01 2.389467e+01 # Hw SPCE + 734 245 4 -0.847600 8.983831e+00 2.446671e+01 2.435894e+01 # Ow SPCE + 735 245 3 0.423800 8.616254e+00 2.382260e+01 2.368811e+01 # Hw SPCE + 736 246 3 0.423800 2.233187e+01 2.661754e+01 9.034738e+00 # Hw SPCE + 737 246 4 -0.847600 2.297202e+01 2.590388e+01 8.750298e+00 # Ow SPCE + 738 246 3 0.423800 2.361025e+01 2.571216e+01 9.495883e+00 # Hw SPCE + 739 247 3 0.423800 6.152311e+00 1.591896e+01 1.583244e+00 # Hw SPCE + 740 247 4 -0.847600 6.458131e+00 1.516173e+01 2.160380e+00 # Ow SPCE + 741 247 3 0.423800 7.447061e+00 1.522371e+01 2.295199e+00 # Hw SPCE + 742 248 3 0.423800 6.249483e+00 8.441998e+00 2.500313e+01 # Hw SPCE + 743 248 4 -0.847600 6.658605e+00 7.854146e+00 2.570102e+01 # Ow SPCE + 744 248 3 0.423800 7.286041e+00 7.207979e+00 2.526652e+01 # Hw SPCE + 745 249 3 0.423800 1.155584e+01 7.485508e+00 4.265325e+00 # Hw SPCE + 746 249 4 -0.847600 1.222371e+01 7.709122e+00 3.555440e+00 # Ow SPCE + 747 249 3 0.423800 1.307003e+01 7.203441e+00 3.722858e+00 # Hw SPCE + 748 250 3 0.423800 6.411808e+00 2.543586e+01 7.998512e+00 # Hw SPCE + 749 250 4 -0.847600 7.212358e+00 2.569130e+01 7.456414e+00 # Ow SPCE + 750 250 3 0.423800 7.538848e+00 2.659058e+01 7.747451e+00 # Hw SPCE + 751 251 3 0.423800 2.686488e+01 1.595570e+01 1.586290e+01 # Hw SPCE + 752 251 4 -0.847600 2.770669e+01 1.542553e+01 1.576147e+01 # Ow SPCE + 753 251 3 0.423800 2.766723e+01 1.489635e+01 1.491388e+01 # Hw SPCE + 754 252 3 0.423800 4.730891e+00 1.312809e+01 1.497769e+00 # Hw SPCE + 755 252 4 -0.847600 4.158177e+00 1.342867e+01 2.260428e+00 # Ow SPCE + 756 252 3 0.423800 3.624401e+00 1.422713e+01 1.981941e+00 # Hw SPCE + 757 253 3 0.423800 2.195203e+01 2.665489e+01 1.887230e+01 # Hw SPCE + 758 253 4 -0.847600 2.258974e+01 2.742501e+01 1.888802e+01 # Ow SPCE + 759 253 3 0.423800 2.352836e+01 2.708031e+01 1.890123e+01 # Hw SPCE + 760 254 3 0.423800 1.253712e+01 3.129762e+00 1.183063e+01 # Hw SPCE + 761 254 4 -0.847600 1.292118e+01 3.990106e+00 1.216575e+01 # Ow SPCE + 762 254 3 0.423800 1.383968e+01 3.827163e+00 1.252605e+01 # Hw SPCE + 763 255 3 0.423800 3.863000e+00 2.523218e+01 2.336537e+01 # Hw SPCE + 764 255 4 -0.847600 3.779878e+00 2.428180e+01 2.306556e+01 # Ow SPCE + 765 255 3 0.423800 4.073781e+00 2.367294e+01 2.380238e+01 # Hw SPCE + 766 256 3 0.423800 1.033123e+01 2.732718e+01 1.991221e+01 # Hw SPCE + 767 256 4 -0.847600 1.083128e+01 2.694936e+01 2.069145e+01 # Ow SPCE + 768 256 3 0.423800 1.037561e+01 2.611750e+01 2.100827e+01 # Hw SPCE + 769 257 3 0.423800 6.606032e+00 3.553686e+00 6.537013e+00 # Hw SPCE + 770 257 4 -0.847600 7.445947e+00 3.035012e+00 6.696762e+00 # Ow SPCE + 771 257 3 0.423800 7.233172e+00 2.208190e+00 7.217426e+00 # Hw SPCE + 772 258 3 0.423800 1.306905e+01 1.800714e+01 6.041763e+00 # Hw SPCE + 773 258 4 -0.847600 1.404572e+01 1.784783e+01 5.897769e+00 # Ow SPCE + 774 258 3 0.423800 1.446986e+01 1.867958e+01 5.539574e+00 # Hw SPCE + 775 259 3 0.423800 6.591547e+00 2.039778e+01 6.554188e+00 # Hw SPCE + 776 259 4 -0.847600 6.892178e+00 1.965746e+01 7.155484e+00 # Ow SPCE + 777 259 3 0.423800 7.891350e+00 1.961712e+01 7.160583e+00 # Hw SPCE + 778 260 3 0.423800 2.102357e+01 1.285810e+01 1.138197e+01 # Hw SPCE + 779 260 4 -0.847600 2.063601e+01 1.330654e+01 1.057655e+01 # Ow SPCE + 780 260 3 0.423800 2.040906e+01 1.425453e+01 1.079974e+01 # Hw SPCE + 781 261 3 0.423800 1.668139e+01 1.365676e+01 7.959222e+00 # Hw SPCE + 782 261 4 -0.847600 1.609109e+01 1.302602e+01 7.455519e+00 # Ow SPCE + 783 261 3 0.423800 1.657944e+01 1.216812e+01 7.295759e+00 # Hw SPCE + 784 262 3 0.423800 1.564369e+01 1.907066e+01 2.197727e+01 # Hw SPCE + 785 262 4 -0.847600 1.642857e+01 1.968608e+01 2.204959e+01 # Ow SPCE + 786 262 3 0.423800 1.726974e+01 1.914825e+01 2.210604e+01 # Hw SPCE + 787 263 3 0.423800 7.519411e+00 1.610097e+01 2.120584e+01 # Hw SPCE + 788 263 4 -0.847600 6.640364e+00 1.562428e+01 2.121221e+01 # Ow SPCE + 789 263 3 0.423800 5.897912e+00 1.629395e+01 2.119468e+01 # Hw SPCE + 790 264 3 0.423800 8.688944e+00 1.522738e+01 1.887923e+01 # Hw SPCE + 791 264 4 -0.847600 9.135790e+00 1.601512e+01 1.845520e+01 # Ow SPCE + 792 264 3 0.423800 1.009324e+01 1.579478e+01 1.826879e+01 # Hw SPCE + 793 265 3 0.423800 2.187599e+01 2.614658e+01 2.179133e+01 # Hw SPCE + 794 265 4 -0.847600 2.260916e+01 2.678722e+01 2.156320e+01 # Ow SPCE + 795 265 3 0.423800 2.235434e+01 2.770623e+01 2.186401e+01 # Hw SPCE + 796 266 3 0.423800 8.584848e+00 1.412426e+01 2.770388e+01 # Hw SPCE + 797 266 4 -0.847600 8.917922e+00 1.350413e+01 2.699360e+01 # Ow SPCE + 798 266 3 0.423800 8.385750e+00 1.363828e+01 2.615766e+01 # Hw SPCE + 799 267 3 0.423800 1.498403e+00 1.136667e+01 1.740482e+01 # Hw SPCE + 800 267 4 -0.847600 1.496704e+00 1.228079e+01 1.699937e+01 # Ow SPCE + 801 267 3 0.423800 1.493879e+00 1.296773e+01 1.772607e+01 # Hw SPCE + 802 268 3 0.423800 2.051142e+01 1.532290e+00 2.720074e+01 # Hw SPCE + 803 268 4 -0.847600 2.144152e+01 1.677620e+00 2.686340e+01 # Ow SPCE + 804 268 3 0.423800 2.143787e+01 1.673320e+00 2.586342e+01 # Hw SPCE + 805 269 3 0.423800 1.074975e+01 1.582297e+01 1.147179e+01 # Hw SPCE + 806 269 4 -0.847600 9.843024e+00 1.620321e+01 1.128940e+01 # Ow SPCE + 807 269 3 0.423800 9.486971e+00 1.582159e+01 1.043641e+01 # Hw SPCE + 808 270 3 0.423800 2.158512e+01 4.175374e+00 2.335242e+01 # Hw SPCE + 809 270 4 -0.847600 2.102097e+01 3.724302e+00 2.404399e+01 # Ow SPCE + 810 270 3 0.423800 2.100196e+01 4.281705e+00 2.487402e+01 # Hw SPCE + 811 271 3 0.423800 1.499420e+00 2.109325e+01 1.083546e+01 # Hw SPCE + 812 271 4 -0.847600 2.014482e+00 2.180659e+01 1.036021e+01 # Ow SPCE + 813 271 3 0.423800 2.990936e+00 2.159428e+01 1.039845e+01 # Hw SPCE + 814 272 3 0.423800 1.593017e+00 7.902730e+00 5.258905e+00 # Hw SPCE + 815 272 4 -0.847600 1.689156e+00 7.147133e+00 4.610962e+00 # Ow SPCE + 816 272 3 0.423800 1.505645e+00 7.476802e+00 3.684873e+00 # Hw SPCE + 817 273 3 0.423800 2.022395e+01 1.561529e+01 1.583840e+01 # Hw SPCE + 818 273 4 -0.847600 1.973203e+01 1.634844e+01 1.536883e+01 # Ow SPCE + 819 273 3 0.423800 1.880786e+01 1.603928e+01 1.514447e+01 # Hw SPCE + 820 274 3 0.423800 1.785576e+00 2.595467e+01 1.540470e+01 # Hw SPCE + 821 274 4 -0.847600 1.929174e+00 2.595737e+01 1.441507e+01 # Ow SPCE + 822 274 3 0.423800 1.499516e+00 2.676845e+01 1.401813e+01 # Hw SPCE + 823 275 3 0.423800 2.471608e+00 1.511177e+01 1.372759e+01 # Hw SPCE + 824 275 4 -0.847600 2.173815e+00 1.544011e+01 1.462398e+01 # Ow SPCE + 825 275 3 0.423800 1.499970e+00 1.616932e+01 1.450486e+01 # Hw SPCE + 826 276 3 0.423800 2.490164e+00 2.322823e+01 1.586321e+01 # Hw SPCE + 827 276 4 -0.847600 2.972830e+00 2.348324e+01 1.502535e+01 # Ow SPCE + 828 276 3 0.423800 3.763314e+00 2.405078e+01 1.525565e+01 # Hw SPCE + 829 277 3 0.423800 1.257057e+01 3.159471e+00 2.191679e+01 # Hw SPCE + 830 277 4 -0.847600 1.321241e+01 3.925963e+00 2.189377e+01 # Ow SPCE + 831 277 3 0.423800 1.270362e+01 4.785319e+00 2.184237e+01 # Hw SPCE + 832 278 3 0.423800 1.172696e+01 1.108398e+01 1.089481e+01 # Hw SPCE + 833 278 4 -0.847600 1.108144e+01 1.033095e+01 1.102229e+01 # Ow SPCE + 834 278 3 0.423800 1.021737e+01 1.055252e+01 1.057030e+01 # Hw SPCE + 835 279 3 0.423800 3.357999e+00 1.799692e+01 1.347523e+01 # Hw SPCE + 836 279 4 -0.847600 4.070804e+00 1.735174e+01 1.375027e+01 # Ow SPCE + 837 279 3 0.423800 4.175658e+00 1.737479e+01 1.474449e+01 # Hw SPCE + 838 280 3 0.423800 2.630102e+01 1.903409e+01 1.695517e+01 # Hw SPCE + 839 280 4 -0.847600 2.716892e+01 1.924741e+01 1.650657e+01 # Ow SPCE + 840 280 3 0.423800 2.727889e+01 1.866650e+01 1.570007e+01 # Hw SPCE + 841 281 3 0.423800 2.251004e+01 6.535603e+00 2.441833e+01 # Hw SPCE + 842 281 4 -0.847600 2.185183e+01 7.272694e+00 2.426517e+01 # Ow SPCE + 843 281 3 0.423800 2.104279e+01 7.120398e+00 2.483285e+01 # Hw SPCE + 844 282 3 0.423800 1.499087e+00 1.589499e+01 2.542090e+01 # Hw SPCE + 845 282 4 -0.847600 1.815620e+00 1.616034e+01 2.451019e+01 # Ow SPCE + 846 282 3 0.423800 2.765466e+00 1.587157e+01 2.439019e+01 # Hw SPCE + 847 283 3 0.423800 7.740973e+00 9.861514e+00 1.493066e+00 # Hw SPCE + 848 283 4 -0.847600 6.754170e+00 9.775212e+00 1.630077e+00 # Ow SPCE + 849 283 3 0.423800 6.577520e+00 9.309741e+00 2.497332e+00 # Hw SPCE + 850 284 3 0.423800 1.061617e+01 1.285220e+01 9.107683e+00 # Hw SPCE + 851 284 4 -0.847600 1.078537e+01 1.346180e+01 9.882119e+00 # Ow SPCE + 852 284 3 0.423800 1.170080e+01 1.385624e+01 9.802028e+00 # Hw SPCE + 853 285 3 0.423800 1.252982e+01 2.440492e+01 2.107781e+01 # Hw SPCE + 854 285 4 -0.847600 1.328350e+01 2.483431e+01 2.157539e+01 # Ow SPCE + 855 285 3 0.423800 1.301279e+01 2.575321e+01 2.186234e+01 # Hw SPCE + 856 286 3 0.423800 1.711392e+01 3.831843e+00 7.910876e+00 # Hw SPCE + 857 286 4 -0.847600 1.745648e+01 2.924441e+00 8.154342e+00 # Ow SPCE + 858 286 3 0.423800 1.668530e+01 2.317735e+00 8.347214e+00 # Hw SPCE + 859 287 3 0.423800 1.418372e+01 2.506933e+01 2.753477e+01 # Hw SPCE + 860 287 4 -0.847600 1.506448e+01 2.551120e+01 2.770510e+01 # Ow SPCE + 861 287 3 0.423800 1.537636e+01 2.596552e+01 2.687064e+01 # Hw SPCE + 862 288 3 0.423800 1.415443e+01 8.808654e+00 1.783263e+01 # Hw SPCE + 863 288 4 -0.847600 1.474094e+01 9.071980e+00 1.859858e+01 # Ow SPCE + 864 288 3 0.423800 1.484475e+01 1.006648e+01 1.861235e+01 # Hw SPCE + 865 289 3 0.423800 4.677556e+00 9.152557e+00 2.714451e+01 # Hw SPCE + 866 289 4 -0.847600 4.161429e+00 9.973113e+00 2.689895e+01 # Ow SPCE + 867 289 3 0.423800 3.680550e+00 1.031808e+01 2.770502e+01 # Hw SPCE + 868 290 3 0.423800 1.681375e+01 5.844343e+00 1.610055e+01 # Hw SPCE + 869 290 4 -0.847600 1.659584e+01 6.638638e+00 1.666766e+01 # Ow SPCE + 870 290 3 0.423800 1.560520e+01 6.774046e+00 1.668509e+01 # Hw SPCE + 871 291 3 0.423800 2.156748e+01 3.556166e+00 8.743084e+00 # Hw SPCE + 872 291 4 -0.847600 2.099885e+01 4.371413e+00 8.633380e+00 # Ow SPCE + 873 291 3 0.423800 2.011280e+01 4.110671e+00 8.250059e+00 # Hw SPCE + 874 292 3 0.423800 2.407964e+00 3.416603e+00 4.875553e+00 # Hw SPCE + 875 292 4 -0.847600 2.941172e+00 4.243659e+00 5.053503e+00 # Ow SPCE + 876 292 3 0.423800 3.734200e+00 4.013998e+00 5.617739e+00 # Hw SPCE + 877 293 3 0.423800 5.101947e+00 1.984796e+01 1.813229e+01 # Hw SPCE + 878 293 4 -0.847600 4.344155e+00 1.945528e+01 1.865340e+01 # Ow SPCE + 879 293 3 0.423800 4.223615e+00 1.849669e+01 1.839539e+01 # Hw SPCE + 880 294 3 0.423800 2.200936e+01 1.060534e+01 3.752114e+00 # Hw SPCE + 881 294 4 -0.847600 2.189049e+01 9.647989e+00 4.015453e+00 # Ow SPCE + 882 294 3 0.423800 2.092346e+01 9.401882e+00 3.949942e+00 # Hw SPCE + 883 295 3 0.423800 1.906823e+01 2.481498e+01 1.981312e+01 # Hw SPCE + 884 295 4 -0.847600 2.004704e+01 2.462769e+01 1.973036e+01 # Ow SPCE + 885 295 3 0.423800 2.020559e+01 2.364324e+01 1.980589e+01 # Hw SPCE + 886 296 3 0.423800 8.611320e+00 2.469462e+01 1.622672e+01 # Hw SPCE + 887 296 4 -0.847600 9.439002e+00 2.491431e+01 1.674313e+01 # Ow SPCE + 888 296 3 0.423800 9.564199e+00 2.590618e+01 1.676596e+01 # Hw SPCE + 889 297 3 0.423800 1.450721e+01 2.761357e+00 1.498414e+01 # Hw SPCE + 890 297 4 -0.847600 1.415911e+01 3.378652e+00 1.568967e+01 # Ow SPCE + 891 297 3 0.423800 1.492505e+01 3.846449e+00 1.613071e+01 # Hw SPCE + 892 298 3 0.423800 1.868170e+01 1.216338e+01 2.369258e+01 # Hw SPCE + 893 298 4 -0.847600 1.793724e+01 1.165537e+01 2.412583e+01 # Ow SPCE + 894 298 3 0.423800 1.733910e+01 1.127446e+01 2.342075e+01 # Hw SPCE + 895 299 3 0.423800 1.579916e+01 1.257851e+01 4.190722e+00 # Hw SPCE + 896 299 4 -0.847600 1.647864e+01 1.330728e+01 4.105902e+00 # Ow SPCE + 897 299 3 0.423800 1.738136e+01 1.290518e+01 3.952950e+00 # Hw SPCE + 898 300 3 0.423800 8.783834e+00 4.391999e+00 2.574931e+01 # Hw SPCE + 899 300 4 -0.847600 8.834362e+00 3.580977e+00 2.633214e+01 # Ow SPCE + 900 300 3 0.423800 9.303500e+00 3.811808e+00 2.718456e+01 # Hw SPCE + 901 301 3 0.423800 6.011865e+00 9.555313e+00 7.226940e+00 # Hw SPCE + 902 301 4 -0.847600 5.112292e+00 9.315952e+00 7.592283e+00 # Ow SPCE + 903 301 3 0.423800 5.076259e+00 9.543963e+00 8.565274e+00 # Hw SPCE + 904 302 3 0.423800 2.627737e+01 1.835769e+01 1.705721e+00 # Hw SPCE + 905 302 4 -0.847600 2.714217e+01 1.785579e+01 1.720783e+00 # Ow SPCE + 906 302 3 0.423800 2.696433e+01 1.689110e+01 1.915100e+00 # Hw SPCE + 907 303 3 0.423800 3.574592e+00 1.061416e+01 2.280540e+01 # Hw SPCE + 908 303 4 -0.847600 3.055829e+00 9.761396e+00 2.274473e+01 # Ow SPCE + 909 303 3 0.423800 2.083376e+00 9.969569e+00 2.263986e+01 # Hw SPCE + 910 304 3 0.423800 6.296561e+00 4.609936e+00 3.590033e+00 # Hw SPCE + 911 304 4 -0.847600 5.307980e+00 4.579235e+00 3.442501e+00 # Ow SPCE + 912 304 3 0.423800 4.921826e+00 5.491212e+00 3.581000e+00 # Hw SPCE + 913 305 3 0.423800 2.206674e+01 2.595682e+01 2.671382e+01 # Hw SPCE + 914 305 4 -0.847600 2.135678e+01 2.658763e+01 2.702694e+01 # Ow SPCE + 915 305 3 0.423800 2.078094e+01 2.613123e+01 2.770524e+01 # Hw SPCE + 916 306 3 0.423800 1.817609e+01 2.098444e+01 2.027086e+01 # Hw SPCE + 917 306 4 -0.847600 1.916371e+01 2.114133e+01 2.027198e+01 # Ow SPCE + 918 306 3 0.423800 1.957493e+01 2.067163e+01 2.105318e+01 # Hw SPCE + 919 307 3 0.423800 1.508206e+01 2.274541e+01 1.655332e+01 # Hw SPCE + 920 307 4 -0.847600 1.463912e+01 2.352206e+01 1.610542e+01 # Ow SPCE + 921 307 3 0.423800 1.476799e+01 2.434437e+01 1.665968e+01 # Hw SPCE + 922 308 3 0.423800 1.860141e+01 7.614679e+00 1.499272e+01 # Hw SPCE + 923 308 4 -0.847600 1.949309e+01 7.194380e+00 1.516084e+01 # Ow SPCE + 924 308 3 0.423800 1.938985e+01 6.200796e+00 1.520703e+01 # Hw SPCE + 925 309 3 0.423800 2.555072e+01 1.158947e+01 2.770860e+01 # Hw SPCE + 926 309 4 -0.847600 2.643932e+01 1.113079e+01 2.770565e+01 # Ow SPCE + 927 309 3 0.423800 2.716796e+01 1.181567e+01 2.770996e+01 # Hw SPCE + 928 310 3 0.423800 1.722971e+01 1.047192e+01 5.228288e+00 # Hw SPCE + 929 310 4 -0.847600 1.777259e+01 9.727661e+00 4.839246e+00 # Ow SPCE + 930 310 3 0.423800 1.859163e+01 9.586550e+00 5.395364e+00 # Hw SPCE + 931 311 3 0.423800 2.587903e+01 2.770480e+01 1.071070e+01 # Hw SPCE + 932 311 4 -0.847600 2.587408e+01 2.767484e+01 9.711163e+00 # Ow SPCE + 933 311 3 0.423800 2.681437e+01 2.762443e+01 9.374545e+00 # Hw SPCE + 934 312 3 0.423800 2.683452e+01 2.499904e+00 1.613606e+01 # Hw SPCE + 935 312 4 -0.847600 2.653269e+01 3.436233e+00 1.631547e+01 # Ow SPCE + 936 312 3 0.423800 2.724944e+01 3.928654e+00 1.680922e+01 # Hw SPCE + 937 313 3 0.423800 1.949268e+01 1.128367e+01 2.484942e+00 # Hw SPCE + 938 313 4 -0.847600 2.013505e+01 1.203650e+01 2.341412e+00 # Ow SPCE + 939 313 3 0.423800 1.964783e+01 1.290736e+01 2.406347e+00 # Hw SPCE + 940 314 3 0.423800 7.709374e+00 2.414404e+01 2.771162e+01 # Hw SPCE + 941 314 4 -0.847600 8.447079e+00 2.438436e+01 2.708072e+01 # Ow SPCE + 942 314 3 0.423800 9.057350e+00 2.360032e+01 2.696734e+01 # Hw SPCE + 943 315 3 0.423800 2.770494e+01 2.154717e+00 2.400702e+01 # Hw SPCE + 944 315 4 -0.847600 2.722867e+01 2.468793e+00 2.482831e+01 # Ow SPCE + 945 315 3 0.423800 2.767391e+01 2.087109e+00 2.563830e+01 # Hw SPCE + 946 316 3 0.423800 2.312652e+01 2.631054e+01 1.499737e+00 # Hw SPCE + 947 316 4 -0.847600 2.283269e+01 2.535561e+01 1.541925e+00 # Ow SPCE + 948 316 3 0.423800 2.209421e+01 2.526330e+01 2.209853e+00 # Hw SPCE + 949 317 3 0.423800 3.512223e+00 1.506331e+01 2.059379e+01 # Hw SPCE + 950 317 4 -0.847600 2.579657e+00 1.521266e+01 2.026513e+01 # Ow SPCE + 951 317 3 0.423800 1.995243e+00 1.548068e+01 2.103105e+01 # Hw SPCE + 952 318 3 0.423800 2.429693e+01 7.441439e+00 8.683691e+00 # Hw SPCE + 953 318 4 -0.847600 2.381138e+01 6.574628e+00 8.570202e+00 # Ow SPCE + 954 318 3 0.423800 2.314115e+01 6.662026e+00 7.833213e+00 # Hw SPCE + 955 319 3 0.423800 7.522703e+00 2.737277e+01 3.043788e+00 # Hw SPCE + 956 319 4 -0.847600 8.363778e+00 2.691935e+01 3.338753e+00 # Ow SPCE + 957 319 3 0.423800 9.106122e+00 2.758810e+01 3.380004e+00 # Hw SPCE + 958 320 3 0.423800 1.882098e+01 5.571858e+00 2.228557e+01 # Hw SPCE + 959 320 4 -0.847600 1.938648e+01 5.803385e+00 2.307715e+01 # Ow SPCE + 960 320 3 0.423800 1.882965e+01 5.766062e+00 2.390694e+01 # Hw SPCE + 961 321 3 0.423800 2.349594e+01 1.890656e+01 2.187820e+01 # Hw SPCE + 962 321 4 -0.847600 2.438719e+01 1.922899e+01 2.155931e+01 # Ow SPCE + 963 321 3 0.423800 2.446815e+01 1.906638e+01 2.057594e+01 # Hw SPCE + 964 322 3 0.423800 9.536260e+00 5.620589e+00 4.385983e+00 # Hw SPCE + 965 322 4 -0.847600 9.017970e+00 4.765629e+00 4.365479e+00 # Ow SPCE + 966 322 3 0.423800 9.365801e+00 4.147994e+00 5.070843e+00 # Hw SPCE + 967 323 3 0.423800 6.563604e+00 1.144852e+01 5.310816e+00 # Hw SPCE + 968 323 4 -0.847600 7.395750e+00 1.159137e+01 5.846659e+00 # Ow SPCE + 969 323 3 0.423800 7.176574e+00 1.211622e+01 6.669152e+00 # Hw SPCE + 970 324 3 0.423800 1.966990e+01 1.748868e+01 1.152281e+01 # Hw SPCE + 971 324 4 -0.847600 2.032778e+01 1.685289e+01 1.192649e+01 # Ow SPCE + 972 324 3 0.423800 1.986999e+01 1.629389e+01 1.261783e+01 # Hw SPCE + 973 325 3 0.423800 1.747869e+01 1.936643e+01 1.586861e+01 # Hw SPCE + 974 325 4 -0.847600 1.772310e+01 1.843509e+01 1.613854e+01 # Ow SPCE + 975 325 3 0.423800 1.692776e+01 1.783807e+01 1.603358e+01 # Hw SPCE + 976 326 3 0.423800 7.512513e+00 6.837798e+00 1.561602e+00 # Hw SPCE + 977 326 4 -0.847600 7.975365e+00 5.953103e+00 1.506077e+00 # Ow SPCE + 978 326 3 0.423800 8.941306e+00 6.067128e+00 1.738364e+00 # Hw SPCE + 979 327 3 0.423800 1.576153e+01 1.524842e+00 5.104682e+00 # Hw SPCE + 980 327 4 -0.847600 1.648750e+01 2.206265e+00 5.011799e+00 # Ow SPCE + 981 327 3 0.423800 1.619187e+01 3.066886e+00 5.426446e+00 # Hw SPCE + 982 328 3 0.423800 1.041282e+01 1.861899e+01 5.679153e+00 # Hw SPCE + 983 328 4 -0.847600 1.039014e+01 1.838326e+01 4.707600e+00 # Ow SPCE + 984 328 3 0.423800 1.080342e+01 1.748259e+01 4.573414e+00 # Hw SPCE + 985 329 3 0.423800 1.662912e+01 1.960472e+01 1.057467e+01 # Hw SPCE + 986 329 4 -0.847600 1.568158e+01 1.987224e+01 1.074964e+01 # Ow SPCE + 987 329 3 0.423800 1.512288e+01 1.905336e+01 1.088108e+01 # Hw SPCE + 988 330 3 0.423800 2.562112e+01 2.330374e+00 2.186263e+01 # Hw SPCE + 989 330 4 -0.847600 2.639694e+01 2.945170e+00 2.172070e+01 # Ow SPCE + 990 330 3 0.423800 2.613489e+01 3.671241e+00 2.108498e+01 # Hw SPCE + 991 331 3 0.423800 2.448963e+01 1.656753e+01 2.459110e+01 # Hw SPCE + 992 331 4 -0.847600 2.532168e+01 1.702758e+01 2.490103e+01 # Ow SPCE + 993 331 3 0.423800 2.611790e+01 1.646961e+01 2.466717e+01 # Hw SPCE + 994 332 3 0.423800 6.417758e+00 2.435520e+01 1.460340e+01 # Hw SPCE + 995 332 4 -0.847600 6.300358e+00 2.524145e+01 1.415532e+01 # Ow SPCE + 996 332 3 0.423800 7.192182e+00 2.558969e+01 1.386657e+01 # Hw SPCE + 997 333 3 0.423800 2.759772e+01 1.733045e+00 3.198930e+00 # Hw SPCE + 998 333 4 -0.847600 2.733255e+01 1.589050e+00 2.245541e+00 # Ow SPCE + 999 333 3 0.423800 2.681386e+01 2.379914e+00 1.920750e+00 # Hw SPCE + 1000 334 3 0.423800 2.588019e+01 1.500059e+00 1.857574e+01 # Hw SPCE + 1001 334 4 -0.847600 2.521085e+01 2.077152e+00 1.904366e+01 # Ow SPCE + 1002 334 3 0.423800 2.452203e+01 1.500061e+00 1.948240e+01 # Hw SPCE + 1003 335 3 0.423800 2.129430e+01 2.706783e+01 2.431980e+01 # Hw SPCE + 1004 335 4 -0.847600 2.048933e+01 2.761599e+01 2.454682e+01 # Ow SPCE + 1005 335 3 0.423800 1.990824e+01 2.770571e+01 2.373794e+01 # Hw SPCE + 1006 336 3 0.423800 7.846173e+00 1.094805e+01 9.044554e+00 # Hw SPCE + 1007 336 4 -0.847600 8.350386e+00 1.031715e+01 8.454855e+00 # Ow SPCE + 1008 336 3 0.423800 9.135128e+00 1.079028e+01 8.054438e+00 # Hw SPCE + 1009 337 3 0.423800 1.813806e+01 1.135820e+01 1.530019e+01 # Hw SPCE + 1010 337 4 -0.847600 1.803468e+01 1.102175e+01 1.436418e+01 # Ow SPCE + 1011 337 3 0.423800 1.870041e+01 1.029479e+01 1.419585e+01 # Hw SPCE + 1012 338 3 0.423800 1.930193e+01 1.912951e+00 3.422805e+00 # Hw SPCE + 1013 338 4 -0.847600 2.017995e+01 1.724286e+00 3.862672e+00 # Ow SPCE + 1014 338 3 0.423800 2.082751e+01 2.453171e+00 3.640448e+00 # Hw SPCE + 1015 339 3 0.423800 4.001940e+00 8.178245e+00 1.133370e+01 # Hw SPCE + 1016 339 4 -0.847600 3.264545e+00 8.098159e+00 1.066300e+01 # Ow SPCE + 1017 339 3 0.423800 3.600575e+00 7.615500e+00 9.854222e+00 # Hw SPCE + 1018 340 3 0.423800 1.659027e+01 2.182766e+01 6.306849e+00 # Hw SPCE + 1019 340 4 -0.847600 1.573368e+01 2.131770e+01 6.228114e+00 # Ow SPCE + 1020 340 3 0.423800 1.525717e+01 2.159339e+01 5.393290e+00 # Hw SPCE + 1021 341 3 0.423800 1.307614e+01 1.247914e+01 2.352415e+01 # Hw SPCE + 1022 341 4 -0.847600 1.282345e+01 1.342121e+01 2.374472e+01 # Ow SPCE + 1023 341 3 0.423800 1.200319e+01 1.367502e+01 2.323212e+01 # Hw SPCE + 1024 342 3 0.423800 1.497260e+00 1.394711e+01 5.663732e+00 # Hw SPCE + 1025 342 4 -0.847600 1.700412e+00 1.368341e+01 4.720762e+00 # Ow SPCE + 1026 342 3 0.423800 2.683724e+00 1.376262e+01 4.556984e+00 # Hw SPCE + 1027 343 3 0.423800 1.331392e+01 4.053633e+00 8.310192e+00 # Hw SPCE + 1028 343 4 -0.847600 1.396490e+01 4.044457e+00 7.551157e+00 # Ow SPCE + 1029 343 3 0.423800 1.422587e+01 3.100890e+00 7.347255e+00 # Hw SPCE + 1030 344 3 0.423800 1.981757e+01 1.143084e+01 8.740400e+00 # Hw SPCE + 1031 344 4 -0.847600 2.013820e+01 1.062139e+01 9.232317e+00 # Ow SPCE + 1032 344 3 0.423800 2.106600e+01 1.078181e+01 9.569139e+00 # Hw SPCE + 1033 345 3 0.423800 1.322045e+01 1.551728e+01 1.471662e+01 # Hw SPCE + 1034 345 4 -0.847600 1.356744e+01 1.622003e+01 1.533770e+01 # Ow SPCE + 1035 345 3 0.423800 1.280432e+01 1.676712e+01 1.568170e+01 # Hw SPCE + 1036 346 3 0.423800 2.397971e+01 1.596693e+01 4.479312e+00 # Hw SPCE + 1037 346 4 -0.847600 2.394370e+01 1.692386e+01 4.191220e+00 # Ow SPCE + 1038 346 3 0.423800 2.474897e+01 1.713577e+01 3.637489e+00 # Hw SPCE + 1039 347 3 0.423800 2.621039e+01 2.495880e+01 9.979554e+00 # Hw SPCE + 1040 347 4 -0.847600 2.717528e+01 2.491488e+01 9.720619e+00 # Ow SPCE + 1041 347 3 0.423800 2.770734e+01 2.453968e+01 1.047966e+01 # Hw SPCE + 1042 348 3 0.423800 1.496218e+01 1.063514e+01 2.219722e+01 # Hw SPCE + 1043 348 4 -0.847600 1.527717e+01 9.890283e+00 2.160902e+01 # Ow SPCE + 1044 348 3 0.423800 1.616433e+01 1.012956e+01 2.121444e+01 # Hw SPCE + 1045 349 3 0.423800 5.054437e+00 2.163300e+01 1.503551e+01 # Hw SPCE + 1046 349 4 -0.847600 4.322602e+00 2.106058e+01 1.466572e+01 # Ow SPCE + 1047 349 3 0.423800 4.564201e+00 2.009674e+01 1.477821e+01 # Hw SPCE + 1048 350 3 0.423800 1.707807e+01 2.164478e+01 1.236295e+01 # Hw SPCE + 1049 350 4 -0.847600 1.667106e+01 2.225955e+01 1.168739e+01 # Ow SPCE + 1050 350 3 0.423800 1.611709e+01 2.294854e+01 1.215474e+01 # Hw SPCE + 1051 351 3 0.423800 6.976035e+00 1.779520e+01 1.276337e+01 # Hw SPCE + 1052 351 4 -0.847600 6.523152e+00 1.699626e+01 1.236766e+01 # Ow SPCE + 1053 351 3 0.423800 7.208840e+00 1.630862e+01 1.212892e+01 # Hw SPCE + 1054 352 3 0.423800 2.164730e+01 2.694246e+01 1.255154e+01 # Hw SPCE + 1055 352 4 -0.847600 2.229751e+01 2.618901e+01 1.245390e+01 # Ow SPCE + 1056 352 3 0.423800 2.315706e+01 2.653692e+01 1.207954e+01 # Hw SPCE + 1057 353 3 0.423800 6.833277e+00 2.770802e+01 1.645841e+01 # Hw SPCE + 1058 353 4 -0.847600 7.004922e+00 2.688481e+01 1.699957e+01 # Ow SPCE + 1059 353 3 0.423800 6.350781e+00 2.617384e+01 1.674145e+01 # Hw SPCE + 1060 354 3 0.423800 1.839299e+01 2.770356e+01 2.009264e+01 # Hw SPCE + 1061 354 4 -0.847600 1.905118e+01 2.770991e+01 2.084546e+01 # Ow SPCE + 1062 354 3 0.423800 1.998036e+01 2.770478e+01 2.047586e+01 # Hw SPCE + 1063 355 3 0.423800 2.064587e+01 1.196502e+01 2.770409e+01 # Hw SPCE + 1064 355 4 -0.847600 2.115792e+01 1.209057e+01 2.685436e+01 # Ow SPCE + 1065 355 3 0.423800 2.178395e+01 1.286403e+01 2.695363e+01 # Hw SPCE + 1066 356 3 0.423800 2.679484e+01 2.239034e+01 1.193203e+01 # Hw SPCE + 1067 356 4 -0.847600 2.667630e+01 2.273676e+01 1.286259e+01 # Ow SPCE + 1068 356 3 0.423800 2.726684e+01 2.353199e+01 1.299996e+01 # Hw SPCE + 1069 357 3 0.423800 2.770507e+01 1.752615e+01 1.228536e+01 # Hw SPCE + 1070 357 4 -0.847600 2.749083e+01 1.721084e+01 1.320985e+01 # Ow SPCE + 1071 357 3 0.423800 2.659323e+01 1.677004e+01 1.321203e+01 # Hw SPCE + 1072 358 3 0.423800 7.054104e+00 1.410756e+01 4.776771e+00 # Hw SPCE + 1073 358 4 -0.847600 8.051952e+00 1.404602e+01 4.799402e+00 # Ow SPCE + 1074 358 3 0.423800 8.434858e+00 1.493321e+01 5.056846e+00 # Hw SPCE + 1075 359 3 0.423800 7.117855e+00 1.158648e+01 2.086169e+01 # Hw SPCE + 1076 359 4 -0.847600 7.039453e+00 1.242040e+01 2.031541e+01 # Ow SPCE + 1077 359 3 0.423800 7.832717e+00 1.249916e+01 1.971165e+01 # Hw SPCE + 1078 360 3 0.423800 2.261313e+01 1.962782e+01 4.727741e+00 # Hw SPCE + 1079 360 4 -0.847600 2.305377e+01 1.917276e+01 5.501535e+00 # Ow SPCE + 1080 360 3 0.423800 2.246091e+01 1.923188e+01 6.304662e+00 # Hw SPCE + 1081 361 3 0.423800 2.644084e+00 2.634256e+01 1.150517e+01 # Hw SPCE + 1082 361 4 -0.847600 2.636942e+00 2.733967e+01 1.158078e+01 # Ow SPCE + 1083 361 3 0.423800 1.836409e+00 2.770428e+01 1.110516e+01 # Hw SPCE + 1084 362 3 0.423800 8.040783e+00 1.358334e+01 9.796191e+00 # Hw SPCE + 1085 362 4 -0.847600 7.086989e+00 1.388373e+01 9.802364e+00 # Ow SPCE + 1086 362 3 0.423800 6.576055e+00 1.338543e+01 9.101906e+00 # Hw SPCE + 1087 363 3 0.423800 2.619651e+00 1.319647e+01 2.242900e+01 # Hw SPCE + 1088 363 4 -0.847600 1.972508e+00 1.282938e+01 2.309717e+01 # Ow SPCE + 1089 363 3 0.423800 1.494472e+00 1.358356e+01 2.354737e+01 # Hw SPCE + 1090 364 3 0.423800 4.287560e+00 6.326031e+00 2.681750e+01 # Hw SPCE + 1091 364 4 -0.847600 4.221751e+00 5.858179e+00 2.593615e+01 # Ow SPCE + 1092 364 3 0.423800 4.606223e+00 4.938624e+00 2.601739e+01 # Hw SPCE + 1093 365 3 0.423800 2.134184e+01 1.704290e+01 5.072956e+00 # Hw SPCE + 1094 365 4 -0.847600 2.040579e+01 1.710436e+01 4.726504e+00 # Ow SPCE + 1095 365 3 0.423800 2.037017e+01 1.673983e+01 3.795993e+00 # Hw SPCE + 1096 366 3 0.423800 1.898318e+01 2.671834e+01 1.475007e+01 # Hw SPCE + 1097 366 4 -0.847600 1.928304e+01 2.767034e+01 1.481164e+01 # Ow SPCE + 1098 366 3 0.423800 2.028241e+01 2.770555e+01 1.481376e+01 # Hw SPCE + 1099 367 3 0.423800 2.348205e+01 1.705352e+01 2.743345e+01 # Hw SPCE + 1100 367 4 -0.847600 2.446126e+01 1.722463e+01 2.754240e+01 # Ow SPCE + 1101 367 3 0.423800 2.461312e+01 1.819940e+01 2.770595e+01 # Hw SPCE + 1102 368 3 0.423800 2.430373e+01 1.513396e+01 1.529098e+01 # Hw SPCE + 1103 368 4 -0.847600 2.387002e+01 1.513478e+01 1.619203e+01 # Ow SPCE + 1104 368 3 0.423800 2.295803e+01 1.553908e+01 1.612260e+01 # Hw SPCE + 1105 369 3 0.423800 1.140164e+01 2.512691e+01 1.865711e+01 # Hw SPCE + 1106 369 4 -0.847600 1.159315e+01 2.415737e+01 1.850437e+01 # Ow SPCE + 1107 369 3 0.423800 1.256364e+01 2.403832e+01 1.829465e+01 # Hw SPCE + 1108 370 3 0.423800 1.903749e+01 1.236107e+01 6.079139e+00 # Hw SPCE + 1109 370 4 -0.847600 2.003143e+01 1.240506e+01 6.179850e+00 # Ow SPCE + 1110 370 3 0.423800 2.044270e+01 1.158083e+01 5.790615e+00 # Hw SPCE + 1111 371 3 0.423800 1.282548e+01 1.241244e+01 1.303471e+01 # Hw SPCE + 1112 371 4 -0.847600 1.310760e+01 1.337123e+01 1.300088e+01 # Ow SPCE + 1113 371 3 0.423800 1.235662e+01 1.392727e+01 1.264472e+01 # Hw SPCE + 1114 372 3 0.423800 1.455246e+01 1.713196e+01 1.777281e+01 # Hw SPCE + 1115 372 4 -0.847600 1.483389e+01 1.618051e+01 1.789743e+01 # Ow SPCE + 1116 372 3 0.423800 1.582185e+01 1.614204e+01 1.804727e+01 # Hw SPCE + 1117 373 3 0.423800 2.660254e+01 1.154715e+01 7.873309e+00 # Hw SPCE + 1118 373 4 -0.847600 2.566434e+01 1.120992e+01 7.951077e+00 # Ow SPCE + 1119 373 3 0.423800 2.566312e+01 1.021159e+01 7.893201e+00 # Hw SPCE + 1120 374 3 0.423800 8.706708e+00 9.237566e+00 1.972409e+01 # Hw SPCE + 1121 374 4 -0.847600 8.843305e+00 8.419627e+00 1.916524e+01 # Ow SPCE + 1122 374 3 0.423800 9.468501e+00 7.795949e+00 1.963445e+01 # Hw SPCE + 1123 375 3 0.423800 1.904385e+01 2.517573e+01 1.240879e+01 # Hw SPCE + 1124 375 4 -0.847600 1.846922e+01 2.435960e+01 1.234762e+01 # Ow SPCE + 1125 375 3 0.423800 1.904403e+01 2.355925e+01 1.217725e+01 # Hw SPCE + 1126 376 3 0.423800 1.112677e+01 1.997104e+01 2.147548e+00 # Hw SPCE + 1127 376 4 -0.847600 1.122781e+01 2.093546e+01 1.903215e+00 # Ow SPCE + 1128 376 3 0.423800 1.219941e+01 2.116740e+01 1.856343e+00 # Hw SPCE + 1129 377 3 0.423800 2.578920e+01 1.183896e+01 1.708350e+01 # Hw SPCE + 1130 377 4 -0.847600 2.566789e+01 1.208960e+01 1.612305e+01 # Ow SPCE + 1131 377 3 0.423800 2.594628e+01 1.304067e+01 1.598904e+01 # Hw SPCE + 1132 378 3 0.423800 1.394495e+01 2.009220e+01 8.623841e+00 # Hw SPCE + 1133 378 4 -0.847600 1.430818e+01 1.923404e+01 8.261025e+00 # Ow SPCE + 1134 378 3 0.423800 1.528668e+01 1.933569e+01 8.081558e+00 # Hw SPCE + 1135 379 3 0.423800 5.844676e+00 2.770452e+01 2.407135e+01 # Hw SPCE + 1136 379 4 -0.847600 6.002067e+00 2.672754e+01 2.421532e+01 # Ow SPCE + 1137 379 3 0.423800 6.928604e+00 2.658707e+01 2.456432e+01 # Hw SPCE + 1138 380 3 0.423800 1.960711e+01 1.884907e+01 1.867877e+01 # Hw SPCE + 1139 380 4 -0.847600 1.959153e+01 1.836640e+01 1.955444e+01 # Ow SPCE + 1140 380 3 0.423800 1.864928e+01 1.830285e+01 1.988328e+01 # Hw SPCE + 1141 381 3 0.423800 1.937001e+01 2.231144e+01 7.504950e+00 # Hw SPCE + 1142 381 4 -0.847600 1.903775e+01 2.136844e+01 7.486339e+00 # Ow SPCE + 1143 381 3 0.423800 1.981507e+01 2.074229e+01 7.425510e+00 # Hw SPCE + 1144 382 3 0.423800 1.186903e+01 7.804165e+00 2.097534e+01 # Hw SPCE + 1145 382 4 -0.847600 1.191707e+01 8.691479e+00 2.051668e+01 # Ow SPCE + 1146 382 3 0.423800 1.284539e+01 9.055616e+00 2.059166e+01 # Hw SPCE + 1147 383 3 0.423800 1.254816e+01 1.752163e+01 2.720322e+01 # Hw SPCE + 1148 383 4 -0.847600 1.171866e+01 1.807176e+01 2.729959e+01 # Ow SPCE + 1149 383 3 0.423800 1.195006e+01 1.895518e+01 2.770705e+01 # Hw SPCE + 1150 384 3 0.423800 2.457569e+01 2.221691e+01 1.644026e+01 # Hw SPCE + 1151 384 4 -0.847600 2.471053e+01 2.126493e+01 1.671514e+01 # Ow SPCE + 1152 384 3 0.423800 2.384588e+01 2.088835e+01 1.704767e+01 # Hw SPCE + 1153 385 3 0.423800 1.283404e+01 2.766561e+01 2.418101e+01 # Hw SPCE + 1154 385 4 -0.847600 1.206202e+01 2.770764e+01 2.354681e+01 # Ow SPCE + 1155 385 3 0.423800 1.120852e+01 2.757858e+01 2.405168e+01 # Hw SPCE + 1156 386 3 0.423800 1.749501e+01 9.243189e+00 2.770432e+01 # Hw SPCE + 1157 386 4 -0.847600 1.707406e+01 8.336104e+00 2.770498e+01 # Ow SPCE + 1158 386 3 0.423800 1.778896e+01 7.636892e+00 2.770954e+01 # Hw SPCE + 1159 387 3 0.423800 3.764698e+00 4.444911e+00 2.206397e+01 # Hw SPCE + 1160 387 4 -0.847600 3.228879e+00 4.580528e+00 2.123060e+01 # Ow SPCE + 1161 387 3 0.423800 3.845459e+00 4.663162e+00 2.044766e+01 # Hw SPCE + 1162 388 3 0.423800 1.825404e+01 5.005912e+00 1.014694e+01 # Hw SPCE + 1163 388 4 -0.847600 1.843498e+01 5.919465e+00 9.782683e+00 # Ow SPCE + 1164 388 3 0.423800 1.764323e+01 6.507321e+00 9.948697e+00 # Hw SPCE + 1165 389 3 0.423800 1.376578e+01 2.484946e+01 1.386117e+01 # Hw SPCE + 1166 389 4 -0.847600 1.385130e+01 2.411884e+01 1.318378e+01 # Ow SPCE + 1167 389 3 0.423800 1.295121e+01 2.391359e+01 1.279947e+01 # Hw SPCE + 1168 390 3 0.423800 6.083757e+00 4.915302e+00 1.185692e+01 # Hw SPCE + 1169 390 4 -0.847600 5.091254e+00 4.926633e+00 1.173522e+01 # Ow SPCE + 1170 390 3 0.423800 4.865042e+00 5.402990e+00 1.088557e+01 # Hw SPCE + 1171 391 3 0.423800 1.088455e+01 2.264855e+01 1.611957e+01 # Hw SPCE + 1172 391 4 -0.847600 1.184680e+01 2.283963e+01 1.592573e+01 # Ow SPCE + 1173 391 3 0.423800 1.194451e+01 2.378898e+01 1.562707e+01 # Hw SPCE + 1174 392 3 0.423800 2.206544e+01 9.962865e+00 2.373843e+01 # Hw SPCE + 1175 392 4 -0.847600 2.203099e+01 1.070683e+01 2.440575e+01 # Ow SPCE + 1176 392 3 0.423800 2.122829e+01 1.127638e+01 2.422883e+01 # Hw SPCE + 1177 393 3 0.423800 2.149681e+00 4.701448e+00 2.459932e+00 # Hw SPCE + 1178 393 4 -0.847600 2.971895e+00 4.426184e+00 1.961740e+00 # Ow SPCE + 1179 393 3 0.423800 3.296291e+00 5.191193e+00 1.405388e+00 # Hw SPCE + 1180 394 3 0.423800 2.234775e+01 2.162751e+01 2.770458e+01 # Hw SPCE + 1181 394 4 -0.847600 2.136405e+01 2.144770e+01 2.770458e+01 # Ow SPCE + 1182 394 3 0.423800 2.086664e+01 2.231522e+01 2.770463e+01 # Hw SPCE + 1183 395 3 0.423800 1.025105e+01 1.356885e+01 1.665922e+01 # Hw SPCE + 1184 395 4 -0.847600 1.035969e+01 1.403984e+01 1.578379e+01 # Ow SPCE + 1185 395 3 0.423800 1.099242e+01 1.352554e+01 1.520487e+01 # Hw SPCE + 1186 396 3 0.423800 1.072202e+01 2.377633e+01 9.484681e+00 # Hw SPCE + 1187 396 4 -0.847600 1.155152e+01 2.385537e+01 1.003758e+01 # Ow SPCE + 1188 396 3 0.423800 1.158705e+01 2.476225e+01 1.045747e+01 # Hw SPCE + 1189 397 3 0.423800 2.341205e+01 1.813661e+01 1.100851e+01 # Hw SPCE + 1190 397 4 -0.847600 2.312469e+01 1.893695e+01 1.153470e+01 # Ow SPCE + 1191 397 3 0.423800 2.213380e+01 1.891079e+01 1.166681e+01 # Hw SPCE + 1192 398 3 0.423800 1.500790e+01 2.394339e+01 3.351174e+00 # Hw SPCE + 1193 398 4 -0.847600 1.452101e+01 2.480831e+01 3.229359e+00 # Ow SPCE + 1194 398 3 0.423800 1.385157e+01 2.492026e+01 3.963743e+00 # Hw SPCE + 1195 399 3 0.423800 1.479502e+01 1.286237e+01 2.563621e+01 # Hw SPCE + 1196 399 4 -0.847600 1.547417e+01 1.230040e+01 2.516403e+01 # Ow SPCE + 1197 399 3 0.423800 1.510530e+01 1.138206e+01 2.502053e+01 # Hw SPCE + 1198 400 3 0.423800 8.068136e+00 1.885540e+01 1.505299e+01 # Hw SPCE + 1199 400 4 -0.847600 7.813925e+00 1.834922e+01 1.587711e+01 # Ow SPCE + 1200 400 3 0.423800 8.632664e+00 1.794760e+01 1.628743e+01 # Hw SPCE + 1201 401 3 0.423800 4.063418e+00 2.228502e+01 2.617453e+01 # Hw SPCE + 1202 401 4 -0.847600 4.103168e+00 2.200493e+01 2.713369e+01 # Ow SPCE + 1203 401 3 0.423800 4.240658e+00 2.281008e+01 2.771061e+01 # Hw SPCE + 1204 402 3 0.423800 1.495103e+00 1.280834e+01 9.475737e+00 # Hw SPCE + 1205 402 4 -0.847600 1.505699e+00 1.300445e+01 8.495213e+00 # Ow SPCE + 1206 402 3 0.423800 2.251321e+00 1.363848e+01 8.290148e+00 # Hw SPCE + 1207 403 3 0.423800 2.465760e+01 6.451081e+00 2.270306e+01 # Hw SPCE + 1208 403 4 -0.847600 2.409442e+01 5.715878e+00 2.232582e+01 # Ow SPCE + 1209 403 3 0.423800 2.432171e+01 4.854913e+00 2.278088e+01 # Hw SPCE + 1210 404 3 0.423800 1.999267e+01 7.260978e+00 2.497131e+00 # Hw SPCE + 1211 404 4 -0.847600 2.085928e+01 6.772162e+00 2.396875e+00 # Ow SPCE + 1212 404 3 0.423800 2.067784e+01 5.804151e+00 2.223575e+00 # Hw SPCE + 1213 405 3 0.423800 9.804016e+00 1.166498e+01 2.144260e+01 # Hw SPCE + 1214 405 4 -0.847600 1.060905e+01 1.108039e+01 2.134174e+01 # Ow SPCE + 1215 405 3 0.423800 1.143569e+01 1.162260e+01 2.149231e+01 # Hw SPCE + 1216 406 3 0.423800 3.092468e+00 1.075047e+01 1.184205e+01 # Hw SPCE + 1217 406 4 -0.847600 2.172613e+00 1.036170e+01 1.189426e+01 # Ow SPCE + 1218 406 3 0.423800 1.497139e+00 1.108044e+01 1.172948e+01 # Hw SPCE + 1219 407 3 0.423800 8.342000e+00 1.946949e+00 1.825736e+01 # Hw SPCE + 1220 407 4 -0.847600 7.967584e+00 2.874061e+00 1.827399e+01 # Ow SPCE + 1221 407 3 0.423800 8.362019e+00 3.379098e+00 1.904168e+01 # Hw SPCE + 1222 408 3 0.423800 1.182612e+01 2.367595e+01 5.346337e+00 # Hw SPCE + 1223 408 4 -0.847600 1.218656e+01 2.331296e+01 6.205591e+00 # Ow SPCE + 1224 408 3 0.423800 1.183999e+01 2.385789e+01 6.969091e+00 # Hw SPCE + 1225 409 3 0.423800 2.018572e+01 8.334075e+00 2.235069e+01 # Hw SPCE + 1226 409 4 -0.847600 1.935425e+01 8.544535e+00 2.286486e+01 # Ow SPCE + 1227 409 3 0.423800 1.940926e+01 9.477685e+00 2.322011e+01 # Hw SPCE + 1228 410 3 0.423800 1.150389e+01 1.904731e+01 2.337866e+01 # Hw SPCE + 1229 410 4 -0.847600 1.246144e+01 1.912402e+01 2.310078e+01 # Ow SPCE + 1230 410 3 0.423800 1.251260e+01 1.953763e+01 2.219177e+01 # Hw SPCE + 1231 411 3 0.423800 1.847815e+01 4.974998e+00 2.770084e+01 # Hw SPCE + 1232 411 4 -0.847600 1.940796e+01 5.342993e+00 2.770581e+01 # Ow SPCE + 1233 411 3 0.423800 2.006473e+01 4.589757e+00 2.767000e+01 # Hw SPCE + 1234 412 3 0.423800 1.580751e+01 1.573107e+01 1.381081e+01 # Hw SPCE + 1235 412 4 -0.847600 1.558902e+01 1.492275e+01 1.326411e+01 # Ow SPCE + 1236 412 3 0.423800 1.643597e+01 1.449518e+01 1.294811e+01 # Hw SPCE + 1237 413 3 0.423800 2.552671e+01 2.186200e+01 4.694186e+00 # Hw SPCE + 1238 413 4 -0.847600 2.590407e+01 2.278594e+01 4.631422e+00 # Ow SPCE + 1239 413 3 0.423800 2.664316e+01 2.279908e+01 3.957950e+00 # Hw SPCE + 1240 414 3 0.423800 1.035260e+01 1.601592e+01 2.375608e+01 # Hw SPCE + 1241 414 4 -0.847600 1.119407e+01 1.631619e+01 2.330691e+01 # Ow SPCE + 1242 414 3 0.423800 1.194221e+01 1.632219e+01 2.397043e+01 # Hw SPCE + 1243 415 3 0.423800 1.366476e+01 2.199974e+01 2.177637e+01 # Hw SPCE + 1244 415 4 -0.847600 1.412014e+01 2.188301e+01 2.265898e+01 # Ow SPCE + 1245 415 3 0.423800 1.408048e+01 2.274131e+01 2.317060e+01 # Hw SPCE + 1246 416 3 0.423800 2.364272e+01 2.771622e+01 1.585868e+01 # Hw SPCE + 1247 416 4 -0.847600 2.381273e+01 2.673639e+01 1.575366e+01 # Ow SPCE + 1248 416 3 0.423800 2.301817e+01 2.630404e+01 1.532735e+01 # Hw SPCE + 1249 417 3 0.423800 9.899911e+00 1.878785e+01 1.222479e+01 # Hw SPCE + 1250 417 4 -0.847600 9.152062e+00 1.943110e+01 1.238898e+01 # Ow SPCE + 1251 417 3 0.423800 9.422298e+00 2.034259e+01 1.207890e+01 # Hw SPCE + 1252 418 3 0.423800 1.878242e+01 4.765640e+00 5.930390e+00 # Hw SPCE + 1253 418 4 -0.847600 1.891751e+01 4.352003e+00 5.030026e+00 # Ow SPCE + 1254 418 3 0.423800 1.970449e+01 4.776921e+00 4.582694e+00 # Hw SPCE + 1255 419 3 0.423800 2.569329e+00 2.015592e+01 1.667337e+01 # Hw SPCE + 1256 419 4 -0.847600 1.639651e+00 1.989432e+01 1.641401e+01 # Ow SPCE + 1257 419 3 0.423800 1.669728e+00 1.906197e+01 1.586059e+01 # Hw SPCE + 1258 420 3 0.423800 1.419487e+01 1.526132e+01 2.687487e+01 # Hw SPCE + 1259 420 4 -0.847600 1.473359e+01 1.511839e+01 2.770514e+01 # Ow SPCE + 1260 420 3 0.423800 1.562750e+01 1.474196e+01 2.746177e+01 # Hw SPCE + 1261 421 3 0.423800 1.225934e+01 4.584227e+00 3.933375e+00 # Hw SPCE + 1262 421 4 -0.847600 1.246279e+01 3.746427e+00 4.440028e+00 # Ow SPCE + 1263 421 3 0.423800 1.333976e+01 3.376246e+00 4.133605e+00 # Hw SPCE + 1264 422 3 0.423800 1.620747e+01 1.499403e+00 1.137835e+01 # Hw SPCE + 1265 422 4 -0.847600 1.549499e+01 1.586802e+00 1.207458e+01 # Ow SPCE + 1266 422 3 0.423800 1.590320e+01 1.493891e+00 1.298273e+01 # Hw SPCE + 1267 423 3 0.423800 9.040061e+00 1.791814e+01 2.669513e+01 # Hw SPCE + 1268 423 4 -0.847600 9.361578e+00 1.786689e+01 2.574961e+01 # Ow SPCE + 1269 423 3 0.423800 9.763981e+00 1.874370e+01 2.548640e+01 # Hw SPCE + 1270 424 3 0.423800 4.419758e+00 1.589273e+01 4.038818e+00 # Hw SPCE + 1271 424 4 -0.847600 3.943488e+00 1.618125e+01 4.869433e+00 # Ow SPCE + 1272 424 3 0.423800 4.594864e+00 1.661046e+01 5.495121e+00 # Hw SPCE + 1273 425 3 0.423800 1.842571e+01 1.489983e+00 1.787524e+01 # Hw SPCE + 1274 425 4 -0.847600 1.758392e+01 1.950976e+00 1.815609e+01 # Ow SPCE + 1275 425 3 0.423800 1.679908e+01 1.499453e+00 1.773164e+01 # Hw SPCE + 1276 426 3 0.423800 9.617069e+00 1.503555e+01 7.768316e+00 # Hw SPCE + 1277 426 4 -0.847600 9.908997e+00 1.580937e+01 7.206183e+00 # Ow SPCE + 1278 426 3 0.423800 1.087713e+01 1.599609e+01 7.373049e+00 # Hw SPCE + 1279 427 3 0.423800 1.761915e+01 1.824001e+01 1.495841e+00 # Hw SPCE + 1280 427 4 -0.847600 1.707860e+01 1.740503e+01 1.598843e+00 # Ow SPCE + 1281 427 3 0.423800 1.610784e+01 1.764023e+01 1.646897e+00 # Hw SPCE + 1282 428 3 0.423800 2.926508e+00 1.529462e+00 2.267458e+01 # Hw SPCE + 1283 428 4 -0.847600 2.327395e+00 2.065396e+00 2.207974e+01 # Ow SPCE + 1284 428 3 0.423800 1.514758e+00 1.528181e+00 2.185386e+01 # Hw SPCE + 1285 429 3 0.423800 1.553029e+01 1.331892e+01 2.261128e+01 # Hw SPCE + 1286 429 4 -0.847600 1.550843e+01 1.423794e+01 2.221768e+01 # Ow SPCE + 1287 429 3 0.423800 1.458388e+01 1.444035e+01 2.189481e+01 # Hw SPCE + 1288 430 3 0.423800 4.535770e+00 7.778783e+00 1.481895e+00 # Hw SPCE + 1289 430 4 -0.847600 3.543503e+00 7.902817e+00 1.486495e+00 # Ow SPCE + 1290 430 3 0.423800 3.329718e+00 8.879692e+00 1.489737e+00 # Hw SPCE + 1291 431 3 0.423800 2.525719e+01 1.568518e+01 1.106507e+01 # Hw SPCE + 1292 431 4 -0.847600 2.557432e+01 1.474391e+01 1.094918e+01 # Ow SPCE + 1293 431 3 0.423800 2.628269e+01 1.454508e+01 1.162645e+01 # Hw SPCE + 1294 432 3 0.423800 1.664803e+01 1.910926e+01 3.917316e+00 # Hw SPCE + 1295 432 4 -0.847600 1.723132e+01 1.835477e+01 4.218178e+00 # Ow SPCE + 1296 432 3 0.423800 1.819067e+01 1.863032e+01 4.157108e+00 # Hw SPCE + 1297 433 3 0.423800 1.358310e+01 2.662050e+01 1.011150e+01 # Hw SPCE + 1298 433 4 -0.847600 1.300630e+01 2.743245e+01 1.002187e+01 # Ow SPCE + 1299 433 3 0.423800 1.296445e+01 2.770719e+01 9.061266e+00 # Hw SPCE + 1300 434 3 0.423800 1.435488e+01 2.080827e+01 2.772371e+01 # Hw SPCE + 1301 434 4 -0.847600 1.506496e+01 2.013414e+01 2.752040e+01 # Ow SPCE + 1302 434 3 0.423800 1.464175e+01 1.930210e+01 2.716177e+01 # Hw SPCE + 1303 435 3 0.423800 2.167525e+01 2.207461e+01 1.584656e+01 # Hw SPCE + 1304 435 4 -0.847600 2.087644e+01 2.148761e+01 1.597828e+01 # Ow SPCE + 1305 435 3 0.423800 2.114010e+01 2.067901e+01 1.650425e+01 # Hw SPCE + 1306 436 3 0.423800 1.336152e+01 8.766368e+00 1.286048e+01 # Hw SPCE + 1307 436 4 -0.847600 1.290502e+01 9.628651e+00 1.307975e+01 # Ow SPCE + 1308 436 3 0.423800 1.199822e+01 9.439351e+00 1.345642e+01 # Hw SPCE + 1309 437 3 0.423800 2.687223e+01 1.379076e+01 4.532287e+00 # Hw SPCE + 1310 437 4 -0.847600 2.610111e+01 1.321002e+01 4.793266e+00 # Ow SPCE + 1311 437 3 0.423800 2.541665e+01 1.376000e+01 5.271855e+00 # Hw SPCE + 1312 438 3 0.423800 2.387246e+01 1.507034e+00 1.669725e+01 # Hw SPCE + 1313 438 4 -0.847600 2.319211e+01 2.086236e+00 1.714629e+01 # Ow SPCE + 1314 438 3 0.423800 2.251393e+01 1.510750e+00 1.760333e+01 # Hw SPCE + 1315 439 3 0.423800 2.653441e+01 1.338040e+01 1.698890e+00 # Hw SPCE + 1316 439 4 -0.847600 2.568911e+01 1.287185e+01 1.862766e+00 # Ow SPCE + 1317 439 3 0.423800 2.578217e+01 1.194475e+01 1.499689e+00 # Hw SPCE + 1318 440 3 0.423800 1.654650e+01 7.351076e+00 4.198674e+00 # Hw SPCE + 1319 440 4 -0.847600 1.720204e+01 6.630419e+00 4.424346e+00 # Ow SPCE + 1320 440 3 0.423800 1.811991e+01 7.022369e+00 4.486733e+00 # Hw SPCE + 1321 441 3 0.423800 1.561748e+01 2.344616e+01 2.579500e+01 # Hw SPCE + 1322 441 4 -0.847600 1.584177e+01 2.276826e+01 2.649512e+01 # Ow SPCE + 1323 441 3 0.423800 1.643484e+01 2.318456e+01 2.718429e+01 # Hw SPCE + 1324 442 3 0.423800 6.712087e+00 1.647363e+01 2.402889e+01 # Hw SPCE + 1325 442 4 -0.847600 7.684457e+00 1.666294e+01 2.389230e+01 # Ow SPCE + 1326 442 3 0.423800 7.794810e+00 1.757954e+01 2.350803e+01 # Hw SPCE + 1327 443 3 0.423800 5.195124e+00 1.484833e+01 1.345621e+01 # Hw SPCE + 1328 443 4 -0.847600 5.562996e+00 1.495140e+01 1.438035e+01 # Ow SPCE + 1329 443 3 0.423800 4.853276e+00 1.472796e+01 1.504847e+01 # Hw SPCE + 1330 444 3 0.423800 8.810832e+00 8.170650e+00 3.650788e+00 # Hw SPCE + 1331 444 4 -0.847600 9.231504e+00 9.063820e+00 3.809800e+00 # Ow SPCE + 1332 444 3 0.423800 8.600516e+00 9.642123e+00 4.326925e+00 # Hw SPCE + 1333 445 3 0.423800 1.481433e+01 2.127865e+01 2.698847e+00 # Hw SPCE + 1334 445 4 -0.847600 1.527661e+01 2.130491e+01 1.812501e+00 # Ow SPCE + 1335 445 3 0.423800 1.564077e+01 2.039825e+01 1.599517e+00 # Hw SPCE + 1336 446 3 0.423800 2.770585e+01 9.745885e+00 6.112819e+00 # Hw SPCE + 1337 446 4 -0.847600 2.769138e+01 8.749499e+00 6.196517e+00 # Ow SPCE + 1338 446 3 0.423800 2.675145e+01 8.420868e+00 6.104123e+00 # Hw SPCE + 1339 447 3 0.423800 1.995729e+01 2.430246e+01 2.248412e+01 # Hw SPCE + 1340 447 4 -0.847600 1.958412e+01 2.507755e+01 2.299401e+01 # Ow SPCE + 1341 447 3 0.423800 1.884602e+01 2.549888e+01 2.246704e+01 # Hw SPCE + 1342 448 3 0.423800 1.497281e+00 1.817499e+01 1.855576e+01 # Hw SPCE + 1343 448 4 -0.847600 1.565131e+00 1.860877e+01 1.945422e+01 # Ow SPCE + 1344 448 3 0.423800 1.511007e+00 1.960184e+01 1.934984e+01 # Hw SPCE + 1345 449 3 0.423800 1.522308e+01 5.358237e+00 1.071042e+01 # Hw SPCE + 1346 449 4 -0.847600 1.552152e+01 4.758830e+00 9.967693e+00 # Ow SPCE + 1347 449 3 0.423800 1.530075e+01 3.810494e+00 1.019556e+01 # Hw SPCE + 1348 450 3 0.423800 2.551853e+01 2.712094e+01 2.391955e+01 # Hw SPCE + 1349 450 4 -0.847600 2.545744e+01 2.641384e+01 2.321508e+01 # Ow SPCE + 1350 450 3 0.423800 2.636236e+01 2.625938e+01 2.281851e+01 # Hw SPCE + 1351 451 3 0.423800 2.385946e+01 4.277050e+00 1.008061e+01 # Hw SPCE + 1352 451 4 -0.847600 2.334099e+01 3.853916e+00 1.082367e+01 # Ow SPCE + 1353 451 3 0.423800 2.397243e+01 3.414585e+00 1.146264e+01 # Hw SPCE + 1354 452 3 0.423800 1.024842e+01 2.543775e+01 1.414465e+01 # Hw SPCE + 1355 452 4 -0.847600 1.115630e+01 2.571591e+01 1.383099e+01 # Ow SPCE + 1356 452 3 0.423800 1.146598e+01 2.650375e+01 1.436336e+01 # Hw SPCE + 1357 453 3 0.423800 1.117636e+01 1.477962e+01 4.921439e+00 # Hw SPCE + 1358 453 4 -0.847600 1.106953e+01 1.411588e+01 4.181144e+00 # Ow SPCE + 1359 453 3 0.423800 1.192150e+01 1.360514e+01 4.065876e+00 # Hw SPCE + 1360 454 3 0.423800 1.155263e+01 2.419941e+01 2.638042e+00 # Hw SPCE + 1361 454 4 -0.847600 1.124181e+01 2.513186e+01 2.453834e+00 # Ow SPCE + 1362 454 3 0.423800 1.199737e+01 2.566636e+01 2.075120e+00 # Hw SPCE + 1363 455 3 0.423800 1.904382e+01 8.719465e+00 1.089114e+01 # Hw SPCE + 1364 455 4 -0.847600 1.879769e+01 8.867065e+00 1.184908e+01 # Ow SPCE + 1365 455 3 0.423800 1.935399e+01 8.274179e+00 1.243132e+01 # Hw SPCE + 1366 456 3 0.423800 2.055866e+01 2.186209e+01 5.026626e+00 # Hw SPCE + 1367 456 4 -0.847600 2.043424e+01 2.280967e+01 4.732347e+00 # Ow SPCE + 1368 456 3 0.423800 1.947974e+01 2.307708e+01 4.864316e+00 # Hw SPCE + 1369 457 3 0.423800 9.488490e+00 2.282360e+01 1.375688e+01 # Hw SPCE + 1370 457 4 -0.847600 1.031447e+01 2.268450e+01 1.321061e+01 # Ow SPCE + 1371 457 3 0.423800 1.029828e+01 2.329670e+01 1.242008e+01 # Hw SPCE + 1372 458 3 0.423800 2.759400e+01 1.561429e+01 1.901357e+01 # Hw SPCE + 1373 458 4 -0.847600 2.771901e+01 1.477632e+01 1.848237e+01 # Ow SPCE + 1374 458 3 0.423800 2.682701e+01 1.442853e+01 1.819361e+01 # Hw SPCE + 1375 459 3 0.423800 9.097112e+00 5.763777e+00 1.055170e+01 # Hw SPCE + 1376 459 4 -0.847600 8.930312e+00 6.009500e+00 1.150658e+01 # Ow SPCE + 1377 459 3 0.423800 9.350086e+00 6.896101e+00 1.170082e+01 # Hw SPCE + 1378 460 3 0.423800 5.431310e+00 2.264163e+01 3.120189e+00 # Hw SPCE + 1379 460 4 -0.847600 5.550080e+00 2.304881e+01 2.214596e+00 # Ow SPCE + 1380 460 3 0.423800 6.336473e+00 2.366641e+01 2.227360e+00 # Hw SPCE + 1381 461 3 0.423800 1.793842e+01 1.486923e+01 5.835118e+00 # Hw SPCE + 1382 461 4 -0.847600 1.846091e+01 1.518650e+01 5.043692e+00 # Ow SPCE + 1383 461 3 0.423800 1.785140e+01 1.566604e+01 4.412394e+00 # Hw SPCE + 1384 462 3 0.423800 2.184585e+00 1.115490e+01 1.453262e+01 # Hw SPCE + 1385 462 4 -0.847600 2.343389e+00 1.213111e+01 1.438500e+01 # Ow SPCE + 1386 462 3 0.423800 1.503737e+00 1.255845e+01 1.404979e+01 # Hw SPCE + 1387 463 3 0.423800 1.441481e+01 3.522791e+00 2.760204e+01 # Hw SPCE + 1388 463 4 -0.847600 1.362346e+01 4.133742e+00 2.762456e+01 # Ow SPCE + 1389 463 3 0.423800 1.393539e+01 5.080332e+00 2.770621e+01 # Hw SPCE + 1390 464 3 0.423800 1.499700e+00 2.188838e+01 2.362087e+01 # Hw SPCE + 1391 464 4 -0.847600 1.945099e+00 2.224829e+01 2.280106e+01 # Ow SPCE + 1392 464 3 0.423800 1.720463e+00 2.166848e+01 2.201789e+01 # Hw SPCE + 1393 465 3 0.423800 2.708400e+01 2.218371e+01 9.205305e+00 # Hw SPCE + 1394 465 4 -0.847600 2.770480e+01 2.226015e+01 8.425070e+00 # Ow SPCE + 1395 465 3 0.423800 2.735466e+01 2.294536e+01 7.786410e+00 # Hw SPCE + 1396 466 3 0.423800 1.333326e+01 1.012770e+01 2.714313e+00 # Hw SPCE + 1397 466 4 -0.847600 1.396010e+01 1.090684e+01 2.710577e+00 # Ow SPCE + 1398 466 3 0.423800 1.343444e+01 1.175750e+01 2.703131e+00 # Hw SPCE + 1399 467 3 0.423800 1.830816e+01 1.219159e+01 1.158591e+01 # Hw SPCE + 1400 467 4 -0.847600 1.755544e+01 1.270612e+01 1.117519e+01 # Ow SPCE + 1401 467 3 0.423800 1.789433e+01 1.322125e+01 1.038792e+01 # Hw SPCE + 1402 468 3 0.423800 2.172042e+01 6.831038e+00 2.770644e+01 # Hw SPCE + 1403 468 4 -0.847600 2.223302e+01 6.193086e+00 2.713176e+01 # Ow SPCE + 1404 468 3 0.423800 2.267323e+01 5.504849e+00 2.770842e+01 # Hw SPCE + 1405 469 3 0.423800 6.268257e+00 1.075389e+01 2.334065e+01 # Hw SPCE + 1406 469 4 -0.847600 7.056234e+00 1.126624e+01 2.368211e+01 # Ow SPCE + 1407 469 3 0.423800 7.274106e+00 1.096339e+01 2.460991e+01 # Hw SPCE + 1408 470 3 0.423800 3.714354e+00 2.000821e+00 1.146332e+01 # Hw SPCE + 1409 470 4 -0.847600 2.897249e+00 1.502381e+00 1.175297e+01 # Ow SPCE + 1410 470 3 0.423800 2.235593e+00 1.482765e+00 1.100342e+01 # Hw SPCE + 1411 471 3 0.423800 2.322907e+01 2.917361e+00 5.997148e+00 # Hw SPCE + 1412 471 4 -0.847600 2.223897e+01 2.777046e+00 5.995080e+00 # Ow SPCE + 1413 471 3 0.423800 2.177665e+01 3.663721e+00 6.003295e+00 # Hw SPCE + 1414 472 3 0.423800 1.763400e+01 1.243773e+01 2.094696e+01 # Hw SPCE + 1415 472 4 -0.847600 1.752266e+01 1.335506e+01 2.056472e+01 # Ow SPCE + 1416 472 3 0.423800 1.837391e+01 1.363744e+01 2.012242e+01 # Hw SPCE + 1417 473 3 0.423800 2.155614e+01 1.654986e+01 2.018860e+01 # Hw SPCE + 1418 473 4 -0.847600 2.107547e+01 1.603549e+01 1.947841e+01 # Ow SPCE + 1419 473 3 0.423800 2.128393e+01 1.642895e+01 1.858301e+01 # Hw SPCE + 1420 474 3 0.423800 1.728050e+01 2.423683e+01 9.871601e+00 # Hw SPCE + 1421 474 4 -0.847600 1.738375e+01 2.347556e+01 9.231428e+00 # Ow SPCE + 1422 474 3 0.423800 1.719527e+01 2.379354e+01 8.302252e+00 # Hw SPCE + 1423 475 3 0.423800 1.650810e+00 6.990915e+00 2.295043e+01 # Hw SPCE + 1424 475 4 -0.847600 2.208121e+00 7.329955e+00 2.370836e+01 # Ow SPCE + 1425 475 3 0.423800 3.171215e+00 7.128177e+00 2.353022e+01 # Hw SPCE + 1426 476 3 0.423800 1.499730e+00 8.992460e+00 8.752017e+00 # Hw SPCE + 1427 476 4 -0.847600 1.510449e+00 9.936633e+00 9.081292e+00 # Ow SPCE + 1428 476 3 0.423800 2.456253e+00 1.025222e+01 9.157852e+00 # Hw SPCE + 1429 477 3 0.423800 1.524357e+00 1.612612e+01 7.337394e+00 # Hw SPCE + 1430 477 4 -0.847600 1.499367e+00 1.628416e+01 8.324511e+00 # Ow SPCE + 1431 477 3 0.423800 1.500368e+00 1.726779e+01 8.504717e+00 # Hw SPCE + 1432 478 3 0.423800 4.508742e+00 2.060049e+01 1.519230e+00 # Hw SPCE + 1433 478 4 -0.847600 5.486779e+00 2.039253e+01 1.505170e+00 # Ow SPCE + 1434 478 3 0.423800 5.616682e+00 1.940102e+01 1.500039e+00 # Hw SPCE + 1435 479 3 0.423800 1.922274e+01 8.093379e+00 8.219502e+00 # Hw SPCE + 1436 479 4 -0.847600 1.916411e+01 7.383197e+00 7.517929e+00 # Ow SPCE + 1437 479 3 0.423800 1.995519e+01 6.775915e+00 7.591466e+00 # Hw SPCE + 1438 480 3 0.423800 1.943784e+01 1.013386e+01 2.597417e+01 # Hw SPCE + 1439 480 4 -0.847600 2.017081e+01 9.453976e+00 2.599674e+01 # Ow SPCE + 1440 480 3 0.423800 2.007103e+01 8.883198e+00 2.681176e+01 # Hw SPCE + 1441 481 3 0.423800 4.192983e+00 2.092863e+01 2.367681e+01 # Hw SPCE + 1442 481 4 -0.847600 3.622627e+00 2.015436e+01 2.340257e+01 # Ow SPCE + 1443 481 3 0.423800 3.162762e+00 1.977780e+01 2.420677e+01 # Hw SPCE + 1444 482 3 0.423800 2.075118e+01 1.717508e+01 2.770677e+01 # Hw SPCE + 1445 482 4 -0.847600 1.976142e+01 1.731709e+01 2.769234e+01 # Ow SPCE + 1446 482 3 0.423800 1.929769e+01 1.643117e+01 2.768228e+01 # Hw SPCE + 1447 483 3 0.423800 6.778056e+00 1.536002e+01 7.209241e+00 # Hw SPCE + 1448 483 4 -0.847600 7.198296e+00 1.626143e+01 7.313492e+00 # Ow SPCE + 1449 483 3 0.423800 6.485256e+00 1.694623e+01 7.463902e+00 # Hw SPCE + 1450 484 3 0.423800 2.425409e+01 5.401110e+00 1.659432e+00 # Hw SPCE + 1451 484 4 -0.847600 2.351087e+01 5.727686e+00 2.243365e+00 # Ow SPCE + 1452 484 3 0.423800 2.360826e+01 6.712456e+00 2.387383e+00 # Hw SPCE + 1453 485 3 0.423800 1.686252e+01 4.577775e+00 1.853409e+01 # Hw SPCE + 1454 485 4 -0.847600 1.717109e+01 5.331094e+00 1.911486e+01 # Ow SPCE + 1455 485 3 0.423800 1.782031e+01 4.985329e+00 1.979233e+01 # Hw SPCE + 1456 486 3 0.423800 2.559447e+01 6.830529e+00 2.554992e+01 # Hw SPCE + 1457 486 4 -0.847600 2.476010e+01 7.328675e+00 2.578589e+01 # Ow SPCE + 1458 486 3 0.423800 2.465753e+01 8.114730e+00 2.517630e+01 # Hw SPCE + 1459 487 3 0.423800 2.618329e+01 1.274776e+01 2.027220e+01 # Hw SPCE + 1460 487 4 -0.847600 2.536296e+01 1.243771e+01 2.075274e+01 # Ow SPCE + 1461 487 3 0.423800 2.554309e+01 1.241004e+01 2.173600e+01 # Hw SPCE + 1462 488 3 0.423800 1.919981e+01 2.661638e+00 2.030213e+01 # Hw SPCE + 1463 488 4 -0.847600 1.973061e+01 2.904574e+00 2.111406e+01 # Ow SPCE + 1464 488 3 0.423800 1.914388e+01 2.856844e+00 2.192244e+01 # Hw SPCE + 1465 489 3 0.423800 2.266744e+01 1.984307e+01 8.978618e+00 # Hw SPCE + 1466 489 4 -0.847600 2.185883e+01 2.037567e+01 9.228600e+00 # Ow SPCE + 1467 489 3 0.423800 2.212644e+01 2.132149e+01 9.412457e+00 # Hw SPCE + 1468 490 3 0.423800 4.817158e+00 1.523570e+00 6.050418e+00 # Hw SPCE + 1469 490 4 -0.847600 4.254228e+00 1.621274e+00 6.871127e+00 # Ow SPCE + 1470 490 3 0.423800 3.291277e+00 1.497598e+00 6.631484e+00 # Hw SPCE + 1471 491 3 0.423800 1.079473e+01 2.139909e+01 1.857674e+01 # Hw SPCE + 1472 491 4 -0.847600 1.177944e+01 2.140440e+01 1.840264e+01 # Ow SPCE + 1473 491 3 0.423800 1.226661e+01 2.114195e+01 1.923559e+01 # Hw SPCE + 1474 492 3 0.423800 2.471848e+01 2.047547e+01 7.260132e+00 # Hw SPCE + 1475 492 4 -0.847600 2.453164e+01 2.145726e+01 7.225866e+00 # Ow SPCE + 1476 492 3 0.423800 2.451936e+01 2.182685e+01 8.154981e+00 # Hw SPCE + 1477 493 3 0.423800 1.868210e+01 2.290815e+01 2.448807e+01 # Hw SPCE + 1478 493 4 -0.847600 1.872346e+01 2.190901e+01 2.448402e+01 # Ow SPCE + 1479 493 3 0.423800 1.929678e+01 2.160222e+01 2.372430e+01 # Hw SPCE + 1480 494 3 0.423800 9.843500e+00 1.140046e+01 2.548708e+01 # Hw SPCE + 1481 494 4 -0.847600 1.054880e+01 1.070283e+01 2.536110e+01 # Ow SPCE + 1482 494 3 0.423800 1.144375e+01 1.114300e+01 2.528825e+01 # Hw SPCE + 1483 495 3 0.423800 6.422993e+00 1.821276e+01 9.903863e+00 # Hw SPCE + 1484 495 4 -0.847600 7.333960e+00 1.850491e+01 9.612681e+00 # Ow SPCE + 1485 495 3 0.423800 8.018713e+00 1.813856e+01 1.024268e+01 # Hw SPCE + 1486 496 3 0.423800 2.471333e+01 2.384780e+01 2.166757e+01 # Hw SPCE + 1487 496 4 -0.847600 2.416185e+01 2.454566e+01 2.121055e+01 # Ow SPCE + 1488 496 3 0.423800 2.476291e+01 2.524546e+01 2.082453e+01 # Hw SPCE + 1489 497 3 0.423800 2.590620e+00 8.091003e+00 2.061080e+01 # Hw SPCE + 1490 497 4 -0.847600 3.231566e+00 7.330610e+00 2.071564e+01 # Ow SPCE + 1491 497 3 0.423800 3.920110e+00 7.372830e+00 1.999168e+01 # Hw SPCE + 1492 498 3 0.423800 8.949464e+00 2.425655e+01 3.695045e+00 # Hw SPCE + 1493 498 4 -0.847600 9.202844e+00 2.412128e+01 4.652907e+00 # Ow SPCE + 1494 498 3 0.423800 9.142579e+00 2.499262e+01 5.139877e+00 # Hw SPCE + 1495 499 3 0.423800 1.764683e+01 2.974023e+00 1.499949e+00 # Hw SPCE + 1496 499 4 -0.847600 1.685020e+01 2.477365e+00 1.844498e+00 # Ow SPCE + 1497 499 3 0.423800 1.705690e+01 1.499622e+00 1.880419e+00 # Hw SPCE + 1498 500 3 0.423800 2.126518e+01 3.913868e+00 1.808240e+01 # Hw SPCE + 1499 500 4 -0.847600 2.045368e+01 3.344904e+00 1.794915e+01 # Ow SPCE + 1500 500 3 0.423800 1.970398e+01 3.909289e+00 1.760359e+01 # Hw SPCE + 1501 501 3 0.423800 1.495876e+00 2.614890e+01 2.082634e+01 # Hw SPCE + 1502 501 4 -0.847600 2.060720e+00 2.545025e+01 2.126548e+01 # Ow SPCE + 1503 501 3 0.423800 1.496549e+00 2.465376e+01 2.148298e+01 # Hw SPCE + 1504 502 3 0.423800 2.279048e+01 9.290222e+00 2.681803e+01 # Hw SPCE + 1505 502 4 -0.847600 2.327017e+01 9.655579e+00 2.761579e+01 # Ow SPCE + 1506 502 3 0.423800 2.416046e+01 9.208611e+00 2.770297e+01 # Hw SPCE + 1507 503 3 0.423800 1.498663e+00 4.051791e+00 1.406843e+01 # Hw SPCE + 1508 503 4 -0.847600 2.227842e+00 4.516378e+00 1.356598e+01 # Ow SPCE + 1509 503 3 0.423800 2.947213e+00 3.859735e+00 1.333943e+01 # Hw SPCE + 1510 504 3 0.423800 4.121509e+00 1.297824e+01 1.026964e+01 # Hw SPCE + 1511 504 4 -0.847600 3.726321e+00 1.373934e+01 1.078398e+01 # Ow SPCE + 1512 504 3 0.423800 3.570218e+00 1.345664e+01 1.173040e+01 # Hw SPCE + 1513 505 3 0.423800 1.938062e+01 1.410367e+01 8.241157e+00 # Hw SPCE + 1514 505 4 -0.847600 1.995758e+01 1.491592e+01 8.155362e+00 # Ow SPCE + 1515 505 3 0.423800 2.066193e+01 1.475299e+01 7.464465e+00 # Hw SPCE + 1516 506 3 0.423800 1.635093e+01 4.629187e+00 2.599280e+01 # Hw SPCE + 1517 506 4 -0.847600 1.660343e+01 5.572751e+00 2.577847e+01 # Ow SPCE + 1518 506 3 0.423800 1.597027e+01 6.198813e+00 2.623361e+01 # Hw SPCE + 1519 507 3 0.423800 2.572095e+01 1.411800e+01 2.664163e+01 # Hw SPCE + 1520 507 4 -0.847600 2.489025e+01 1.466440e+01 2.653495e+01 # Ow SPCE + 1521 507 3 0.423800 2.417927e+01 1.431235e+01 2.714369e+01 # Hw SPCE + 1522 508 3 0.423800 2.106512e+01 3.934685e+00 1.521472e+01 # Hw SPCE + 1523 508 4 -0.847600 2.145982e+01 3.015920e+00 1.522367e+01 # Ow SPCE + 1524 508 3 0.423800 2.073659e+01 2.343985e+00 1.538314e+01 # Hw SPCE + 1525 509 3 0.423800 1.747101e+01 1.089516e+01 1.867742e+01 # Hw SPCE + 1526 509 4 -0.847600 1.750084e+01 1.180616e+01 1.826610e+01 # Ow SPCE + 1527 509 3 0.423800 1.671577e+01 1.192311e+01 1.765784e+01 # Hw SPCE + 1528 510 3 0.423800 2.149389e+01 8.833650e+00 6.611416e+00 # Hw SPCE + 1529 510 4 -0.847600 2.177394e+01 9.611791e+00 7.173615e+00 # Ow SPCE + 1530 510 3 0.423800 2.254669e+01 9.345700e+00 7.749852e+00 # Hw SPCE + 1531 511 3 0.423800 3.791408e+00 1.897825e+01 1.012530e+01 # Hw SPCE + 1532 511 4 -0.847600 3.851687e+00 1.885137e+01 9.135211e+00 # Ow SPCE + 1533 511 3 0.423800 3.179880e+00 1.943767e+01 8.682525e+00 # Hw SPCE + 1534 512 3 0.423800 1.252372e+01 2.047991e+01 2.549174e+01 # Hw SPCE + 1535 512 4 -0.847600 1.324747e+01 2.111712e+01 2.522688e+01 # Ow SPCE + 1536 512 3 0.423800 1.305955e+01 2.201712e+01 2.562020e+01 # Hw SPCE + 1537 513 3 0.423800 9.857980e+00 8.140566e+00 9.299584e+00 # Hw SPCE + 1538 513 4 -0.847600 1.080806e+01 7.847241e+00 9.193255e+00 # Ow SPCE + 1539 513 3 0.423800 1.136372e+01 8.620982e+00 8.889004e+00 # Hw SPCE + 1540 514 3 0.423800 2.468682e+01 1.540730e+01 1.499507e+00 # Hw SPCE + 1541 514 4 -0.847600 2.388089e+01 1.498355e+01 1.912937e+00 # Ow SPCE + 1542 514 3 0.423800 2.305466e+01 1.541498e+01 1.550717e+00 # Hw SPCE + 1543 515 3 0.423800 1.464646e+01 2.252692e+01 1.926250e+01 # Hw SPCE + 1544 515 4 -0.847600 1.547386e+01 2.308846e+01 1.925340e+01 # Ow SPCE + 1545 515 3 0.423800 1.626223e+01 2.251673e+01 1.902626e+01 # Hw SPCE + 1546 516 3 0.423800 2.767962e+01 2.118986e+01 2.645961e+01 # Hw SPCE + 1547 516 4 -0.847600 2.770572e+01 2.108694e+01 2.745396e+01 # Ow SPCE + 1548 516 3 0.423800 2.734198e+01 2.019016e+01 2.770589e+01 # Hw SPCE + 1549 517 3 0.423800 5.473025e+00 1.539565e+01 1.766372e+01 # Hw SPCE + 1550 517 4 -0.847600 5.457163e+00 1.532146e+01 1.866084e+01 # Ow SPCE + 1551 517 3 0.423800 5.719911e+00 1.439500e+01 1.893036e+01 # Hw SPCE + 1552 518 3 0.423800 1.355721e+01 1.609064e+01 7.981517e+00 # Hw SPCE + 1553 518 4 -0.847600 1.451195e+01 1.585086e+01 7.805517e+00 # Ow SPCE + 1554 518 3 0.423800 1.510515e+01 1.659197e+01 8.119965e+00 # Hw SPCE + 1555 519 3 0.423800 1.185686e+01 2.756045e+01 1.746044e+01 # Hw SPCE + 1556 519 4 -0.847600 1.271387e+01 2.750488e+01 1.694815e+01 # Ow SPCE + 1557 519 3 0.423800 1.289860e+01 2.655139e+01 1.670998e+01 # Hw SPCE + 1558 520 3 0.423800 2.770610e+01 1.433506e+01 2.192258e+01 # Hw SPCE + 1559 520 4 -0.847600 2.713948e+01 1.511976e+01 2.217392e+01 # Ow SPCE + 1560 520 3 0.423800 2.770524e+01 1.594383e+01 2.220271e+01 # Hw SPCE + 1561 521 3 0.423800 1.499493e+01 1.238797e+01 2.007948e+01 # Hw SPCE + 1562 521 4 -0.847600 1.464506e+01 1.280227e+01 1.923927e+01 # Ow SPCE + 1563 521 3 0.423800 1.485236e+01 1.378055e+01 1.923864e+01 # Hw SPCE + 1564 522 3 0.423800 1.136191e+01 6.926399e+00 2.399031e+01 # Hw SPCE + 1565 522 4 -0.847600 1.054592e+01 7.370139e+00 2.436079e+01 # Ow SPCE + 1566 522 3 0.423800 9.932311e+00 6.677061e+00 2.473911e+01 # Hw SPCE + 1567 523 3 0.423800 7.548290e+00 2.174968e+00 3.349358e+00 # Hw SPCE + 1568 523 4 -0.847600 6.864919e+00 1.475454e+00 3.558363e+00 # Ow SPCE + 1569 523 3 0.423800 5.952602e+00 1.884488e+00 3.539162e+00 # Hw SPCE + 1570 524 3 0.423800 2.257539e+01 7.886933e+00 2.106609e+01 # Hw SPCE + 1571 524 4 -0.847600 2.236455e+01 7.157236e+00 2.041564e+01 # Ow SPCE + 1572 524 3 0.423800 2.320865e+01 6.845285e+00 1.997954e+01 # Hw SPCE + 1573 525 3 0.423800 1.931716e+01 2.614213e+01 1.741829e+01 # Hw SPCE + 1574 525 4 -0.847600 1.896046e+01 2.707452e+01 1.747657e+01 # Ow SPCE + 1575 525 3 0.423800 1.970938e+01 2.770594e+01 1.767766e+01 # Hw SPCE + 1576 526 3 0.423800 1.293280e+01 1.317757e+01 1.712150e+01 # Hw SPCE + 1577 526 4 -0.847600 1.315759e+01 1.415177e+01 1.710169e+01 # Ow SPCE + 1578 526 3 0.423800 1.254740e+01 1.464728e+01 1.771987e+01 # Hw SPCE + 1579 527 3 0.423800 7.790007e+00 6.046852e+00 2.770758e+01 # Hw SPCE + 1580 527 4 -0.847600 6.829260e+00 5.769427e+00 2.770728e+01 # Ow SPCE + 1581 527 3 0.423800 6.770591e+00 4.771151e+00 2.770535e+01 # Hw SPCE + 1582 528 3 0.423800 1.123375e+01 2.298556e+01 2.380651e+01 # Hw SPCE + 1583 528 4 -0.847600 1.075845e+01 2.212317e+01 2.398080e+01 # Ow SPCE + 1584 528 3 0.423800 1.039779e+01 2.212396e+01 2.491350e+01 # Hw SPCE + 1585 529 3 0.423800 4.200882e+00 1.931622e+01 2.770757e+01 # Hw SPCE + 1586 529 4 -0.847600 5.175713e+00 1.953916e+01 2.770789e+01 # Ow SPCE + 1587 529 3 0.423800 5.710832e+00 1.869439e+01 2.770832e+01 # Hw SPCE + 1588 530 3 0.423800 2.770456e+01 1.697514e+01 9.342626e+00 # Hw SPCE + 1589 530 4 -0.847600 2.770706e+01 1.614854e+01 9.905419e+00 # Ow SPCE + 1590 530 3 0.423800 2.770253e+01 1.534242e+01 9.313697e+00 # Hw SPCE + 1591 531 3 0.423800 1.832884e+01 7.670320e+00 2.009972e+01 # Hw SPCE + 1592 531 4 -0.847600 1.770977e+01 8.379274e+00 1.976188e+01 # Ow SPCE + 1593 531 3 0.423800 1.728273e+01 8.071375e+00 1.891168e+01 # Hw SPCE + 1594 532 3 0.423800 1.164885e+01 2.369859e+00 6.675939e+00 # Hw SPCE + 1595 532 4 -0.847600 1.119973e+01 1.488536e+00 6.822766e+00 # Ow SPCE + 1596 532 3 0.423800 1.021557e+01 1.629733e+00 6.930003e+00 # Hw SPCE + 1597 533 3 0.423800 1.736192e+01 1.608261e+01 2.086977e+01 # Hw SPCE + 1598 533 4 -0.847600 1.663485e+01 1.658639e+01 2.133622e+01 # Ow SPCE + 1599 533 3 0.423800 1.582226e+01 1.661428e+01 2.075406e+01 # Hw SPCE + 1600 534 3 0.423800 2.770538e+01 1.618975e+01 6.697811e+00 # Hw SPCE + 1601 534 4 -0.847600 2.683077e+01 1.586285e+01 6.339763e+00 # Ow SPCE + 1602 534 3 0.423800 2.644970e+01 1.654559e+01 5.716346e+00 # Hw SPCE + 1603 535 3 0.423800 1.073071e+01 3.382764e+00 1.994761e+00 # Hw SPCE + 1604 535 4 -0.847600 1.068786e+01 2.506336e+00 2.474384e+00 # Ow SPCE + 1605 535 3 0.423800 1.061173e+01 1.764903e+00 1.807690e+00 # Hw SPCE + 1606 536 3 0.423800 5.975184e+00 9.418923e+00 1.756302e+01 # Hw SPCE + 1607 536 4 -0.847600 6.920951e+00 9.718084e+00 1.768962e+01 # Ow SPCE + 1608 536 3 0.423800 6.942337e+00 1.070925e+01 1.782049e+01 # Hw SPCE + 1609 537 3 0.423800 1.360924e+01 2.146817e+01 1.387145e+01 # Hw SPCE + 1610 537 4 -0.847600 1.450822e+01 2.119277e+01 1.353090e+01 # Ow SPCE + 1611 537 3 0.423800 1.516476e+01 2.119288e+01 1.428519e+01 # Hw SPCE + 1612 538 3 0.423800 8.874645e+00 2.735638e+00 1.567786e+01 # Hw SPCE + 1613 538 4 -0.847600 8.841213e+00 1.743745e+00 1.555526e+01 # Ow SPCE + 1614 538 3 0.423800 9.281324e+00 1.499682e+00 1.469112e+01 # Hw SPCE + 1615 539 3 0.423800 2.677837e+01 2.727822e+01 1.521683e+01 # Hw SPCE + 1616 539 4 -0.847600 2.716773e+01 2.651889e+01 1.573819e+01 # Ow SPCE + 1617 539 3 0.423800 2.770480e+01 2.593640e+01 1.512805e+01 # Hw SPCE + 1618 540 3 0.423800 1.348931e+01 2.050695e+01 1.644491e+01 # Hw SPCE + 1619 540 4 -0.847600 1.379303e+01 1.955419e+01 1.644174e+01 # Ow SPCE + 1620 540 3 0.423800 1.479253e+01 1.952299e+01 1.643578e+01 # Hw SPCE + 1621 541 3 0.423800 2.712765e+01 2.770458e+01 1.499784e+00 # Hw SPCE + 1622 541 4 -0.847600 2.710752e+01 2.724363e+01 2.386980e+00 # Ow SPCE + 1623 541 3 0.423800 2.619044e+01 2.731545e+01 2.779174e+00 # Hw SPCE + 1624 542 3 0.423800 9.946949e+00 2.110450e+01 2.770454e+01 # Hw SPCE + 1625 542 4 -0.847600 9.160696e+00 2.085784e+01 2.713800e+01 # Ow SPCE + 1626 542 3 0.423800 8.468408e+00 2.041129e+01 2.770486e+01 # Hw SPCE + 1627 543 3 0.423800 1.504399e+00 1.083302e+01 2.009960e+01 # Hw SPCE + 1628 543 4 -0.847600 1.816408e+00 1.177936e+01 2.018384e+01 # Ow SPCE + 1629 543 3 0.423800 2.807855e+00 1.181584e+01 2.005853e+01 # Hw SPCE + 1630 544 3 0.423800 2.543910e+01 2.947611e+00 2.768656e+01 # Hw SPCE + 1631 544 4 -0.847600 2.451490e+01 2.595735e+00 2.753807e+01 # Ow SPCE + 1632 544 3 0.423800 2.455905e+01 1.616936e+00 2.733806e+01 # Hw SPCE + 1633 545 3 0.423800 2.361457e+01 1.312219e+01 1.230861e+01 # Hw SPCE + 1634 545 4 -0.847600 2.413492e+01 1.356531e+01 1.303860e+01 # Ow SPCE + 1635 545 3 0.423800 2.440618e+01 1.288209e+01 1.371657e+01 # Hw SPCE + 1636 546 3 0.423800 1.870442e+01 2.642254e+01 9.001379e+00 # Hw SPCE + 1637 546 4 -0.847600 1.888946e+01 2.680197e+01 8.094851e+00 # Ow SPCE + 1638 546 3 0.423800 1.831281e+01 2.634447e+01 7.417974e+00 # Hw SPCE + 1639 547 3 0.423800 1.955187e+01 2.669861e+01 1.982066e+00 # Hw SPCE + 1640 547 4 -0.847600 1.983701e+01 2.765621e+01 2.023180e+00 # Ow SPCE + 1641 547 3 0.423800 2.083300e+01 2.770447e+01 2.098561e+00 # Hw SPCE + 1642 548 3 0.423800 5.421499e+00 2.757790e+01 1.294457e+01 # Hw SPCE + 1643 548 4 -0.847600 5.110506e+00 2.770464e+01 1.388649e+01 # Ow SPCE + 1644 548 3 0.423800 4.115657e+00 2.761068e+01 1.392453e+01 # Hw SPCE + 1645 549 3 0.423800 1.667681e+01 2.723516e+01 2.218951e+01 # Hw SPCE + 1646 549 4 -0.847600 1.593196e+01 2.771570e+01 2.265241e+01 # Ow SPCE + 1647 549 3 0.423800 1.507418e+01 2.754768e+01 2.216664e+01 # Hw SPCE + 1648 550 3 0.423800 2.433245e+01 2.342839e+01 1.128065e+01 # Hw SPCE + 1649 550 4 -0.847600 2.415467e+01 2.258524e+01 1.077321e+01 # Ow SPCE + 1650 550 3 0.423800 2.415864e+01 2.180938e+01 1.140410e+01 # Hw SPCE + 1651 551 3 0.423800 1.837587e+01 2.123822e+01 2.725995e+01 # Hw SPCE + 1652 551 4 -0.847600 1.779349e+01 2.043021e+01 2.734918e+01 # Ow SPCE + 1653 551 3 0.423800 1.833384e+01 1.966748e+01 2.770453e+01 # Hw SPCE + 1654 552 3 0.423800 1.592706e+01 1.936515e+00 2.435957e+01 # Hw SPCE + 1655 552 4 -0.847600 1.505896e+01 1.499035e+00 2.412505e+01 # Ow SPCE + 1656 552 3 0.423800 1.454993e+01 2.085140e+00 2.349467e+01 # Hw SPCE + 1657 553 3 0.423800 1.747565e+01 1.731294e+01 9.312832e+00 # Hw SPCE + 1658 553 4 -0.847600 1.766196e+01 1.635124e+01 9.111737e+00 # Ow SPCE + 1659 553 3 0.423800 1.809084e+01 1.592158e+01 9.906376e+00 # Hw SPCE + 1660 554 3 0.423800 8.312156e+00 2.203891e+01 1.687596e+01 # Hw SPCE + 1661 554 4 -0.847600 8.145446e+00 2.107334e+01 1.667629e+01 # Ow SPCE + 1662 554 3 0.423800 7.160271e+00 2.090436e+01 1.664666e+01 # Hw SPCE + 1663 555 3 0.423800 1.452310e+01 2.241336e+01 9.977204e+00 # Hw SPCE + 1664 555 4 -0.847600 1.467835e+01 2.336383e+01 9.707951e+00 # Ow SPCE + 1665 555 3 0.423800 1.427486e+01 2.397365e+01 1.039009e+01 # Hw SPCE + 1666 556 3 0.423800 2.770573e+01 9.991646e+00 1.639356e+01 # Hw SPCE + 1667 556 4 -0.847600 2.733485e+01 1.000499e+01 1.546498e+01 # Ow SPCE + 1668 556 3 0.423800 2.770500e+01 9.233753e+00 1.494711e+01 # Hw SPCE + 1669 557 3 0.423800 2.255789e+01 1.833507e+01 1.626641e+01 # Hw SPCE + 1670 557 4 -0.847600 2.351546e+01 1.820880e+01 1.652545e+01 # Ow SPCE + 1671 557 3 0.423800 2.406042e+01 1.802400e+01 1.570760e+01 # Hw SPCE + 1672 558 3 0.423800 1.525572e+01 2.770743e+01 1.200379e+01 # Hw SPCE + 1673 558 4 -0.847600 1.484040e+01 2.707735e+01 1.265992e+01 # Ow SPCE + 1674 558 3 0.423800 1.406834e+01 2.752601e+01 1.311006e+01 # Hw SPCE + 1675 559 3 0.423800 1.658932e+01 1.906188e+01 1.328318e+01 # Hw SPCE + 1676 559 4 -0.847600 1.675201e+01 1.815055e+01 1.290501e+01 # Ow SPCE + 1677 559 3 0.423800 1.758478e+01 1.776758e+01 1.330480e+01 # Hw SPCE + 1678 560 3 0.423800 1.004229e+01 1.712720e+01 2.079854e+01 # Hw SPCE + 1679 560 4 -0.847600 1.063564e+01 1.789587e+01 2.103746e+01 # Ow SPCE + 1680 560 3 0.423800 1.010311e+01 1.874227e+01 2.103197e+01 # Hw SPCE + 1681 561 3 0.423800 2.500505e+01 1.070351e+01 4.512067e+00 # Hw SPCE + 1682 561 4 -0.847600 2.573603e+01 1.010670e+01 4.181178e+00 # Ow SPCE + 1683 561 3 0.423800 2.648741e+01 1.066421e+01 3.828189e+00 # Hw SPCE + 1684 562 3 0.423800 1.210303e+01 1.800383e+01 9.349879e+00 # Hw SPCE + 1685 562 4 -0.847600 1.113080e+01 1.778888e+01 9.442458e+00 # Ow SPCE + 1686 562 3 0.423800 1.060025e+01 1.863653e+01 9.439283e+00 # Hw SPCE + 1687 563 3 0.423800 1.030762e+01 7.414060e+00 2.769564e+01 # Hw SPCE + 1688 563 4 -0.847600 1.049207e+01 6.470196e+00 2.742160e+01 # Ow SPCE + 1689 563 3 0.423800 1.141031e+01 6.209855e+00 2.772004e+01 # Hw SPCE + 1690 564 3 0.423800 4.398211e+00 2.689864e+01 2.124419e+01 # Hw SPCE + 1691 564 4 -0.847600 4.266022e+00 2.770576e+01 2.181959e+01 # Ow SPCE + 1692 564 3 0.423800 3.351717e+00 2.768078e+01 2.222384e+01 # Hw SPCE + 1693 565 3 0.423800 8.283521e+00 2.495708e+01 9.955134e+00 # Hw SPCE + 1694 565 4 -0.847600 7.965370e+00 2.543617e+01 1.077322e+01 # Ow SPCE + 1695 565 3 0.423800 8.600427e+00 2.526670e+01 1.152686e+01 # Hw SPCE + 1696 566 3 0.423800 2.718771e+01 5.401651e+00 2.294482e+01 # Hw SPCE + 1697 566 4 -0.847600 2.770619e+01 6.078594e+00 2.346725e+01 # Ow SPCE + 1698 566 3 0.423800 2.727173e+01 6.974594e+00 2.337542e+01 # Hw SPCE + 1699 567 3 0.423800 1.344671e+01 1.859690e+01 1.608430e+00 # Hw SPCE + 1700 567 4 -0.847600 1.362491e+01 1.880297e+01 2.570605e+00 # Ow SPCE + 1701 567 3 0.423800 1.316244e+01 1.812923e+01 3.146969e+00 # Hw SPCE + 1702 568 3 0.423800 1.583162e+01 2.444348e+01 5.926474e+00 # Hw SPCE + 1703 568 4 -0.847600 1.499075e+01 2.459672e+01 6.445563e+00 # Ow SPCE + 1704 568 3 0.423800 1.478045e+01 2.378288e+01 6.987262e+00 # Hw SPCE + 1705 569 3 0.423800 9.922789e+00 5.193782e+00 2.202243e+01 # Hw SPCE + 1706 569 4 -0.847600 9.563582e+00 6.125784e+00 2.207083e+01 # Ow SPCE + 1707 569 3 0.423800 8.644820e+00 6.149777e+00 2.167675e+01 # Hw SPCE + 1708 570 3 0.423800 8.914720e+00 8.315418e+00 1.641874e+01 # Hw SPCE + 1709 570 4 -0.847600 9.302874e+00 8.063947e+00 1.553212e+01 # Ow SPCE + 1710 570 3 0.423800 1.015658e+01 8.564357e+00 1.538800e+01 # Hw SPCE + 1711 571 3 0.423800 1.175235e+01 1.796050e+01 1.792472e+01 # Hw SPCE + 1712 571 4 -0.847600 1.132726e+01 1.866461e+01 1.849353e+01 # Ow SPCE + 1713 571 3 0.423800 1.034732e+01 1.870417e+01 1.829820e+01 # Hw SPCE + 1714 572 3 0.423800 2.334407e+01 2.469584e+01 1.753610e+01 # Hw SPCE + 1715 572 4 -0.847600 2.274051e+01 2.390140e+01 1.760386e+01 # Ow SPCE + 1716 572 3 0.423800 2.178874e+01 2.420714e+01 1.762968e+01 # Hw SPCE + 1717 573 3 0.423800 1.560477e+01 2.702754e+01 1.716266e+00 # Hw SPCE + 1718 573 4 -0.847600 1.525408e+01 2.737955e+01 2.584082e+00 # Ow SPCE + 1719 573 3 0.423800 1.428469e+01 2.760432e+01 2.485249e+00 # Hw SPCE + 1720 574 3 0.423800 7.381621e+00 2.092978e+01 1.938298e+01 # Hw SPCE + 1721 574 4 -0.847600 8.033211e+00 2.166093e+01 1.958511e+01 # Ow SPCE + 1722 574 3 0.423800 7.576710e+00 2.237804e+01 2.011176e+01 # Hw SPCE + 1723 575 3 0.423800 1.345344e+01 1.285413e+01 6.248165e+00 # Hw SPCE + 1724 575 4 -0.847600 1.335825e+01 1.348350e+01 7.019426e+00 # Ow SPCE + 1725 575 3 0.423800 1.239316e+01 1.371343e+01 7.144852e+00 # Hw SPCE + 1726 576 3 0.423800 1.684996e+01 1.496546e+00 2.112863e+01 # Hw SPCE + 1727 576 4 -0.847600 1.612315e+01 1.500121e+00 2.044180e+01 # Ow SPCE + 1728 576 3 0.423800 1.523334e+01 1.502605e+00 2.089813e+01 # Hw SPCE + 1729 577 3 0.423800 1.259795e+01 6.059961e+00 1.028903e+01 # Hw SPCE + 1730 577 4 -0.847600 1.265203e+01 6.809136e+00 1.094919e+01 # Ow SPCE + 1731 577 3 0.423800 1.210535e+01 6.583051e+00 1.175544e+01 # Hw SPCE + 1732 578 3 0.423800 1.175266e+01 1.642499e+01 1.504706e+00 # Hw SPCE + 1733 578 4 -0.847600 1.124046e+01 1.563939e+01 1.851816e+00 # Ow SPCE + 1734 578 3 0.423800 1.171187e+01 1.479512e+01 1.596895e+00 # Hw SPCE + 1735 579 3 0.423800 1.964008e+01 1.500636e+01 2.214316e+01 # Hw SPCE + 1736 579 4 -0.847600 2.038415e+01 1.503982e+01 2.281043e+01 # Ow SPCE + 1737 579 3 0.423800 2.013831e+01 1.449401e+01 2.361146e+01 # Hw SPCE + 1738 580 3 0.423800 6.359780e+00 1.494462e+00 9.725939e+00 # Hw SPCE + 1739 580 4 -0.847600 5.623160e+00 2.122431e+00 9.474851e+00 # Ow SPCE + 1740 580 3 0.423800 5.796855e+00 3.019459e+00 9.881263e+00 # Hw SPCE + 1741 581 3 0.423800 2.339449e+01 2.716322e+01 4.184562e+00 # Hw SPCE + 1742 581 4 -0.847600 2.365407e+01 2.624102e+01 4.471228e+00 # Ow SPCE + 1743 581 3 0.423800 2.284652e+01 2.575711e+01 4.808419e+00 # Hw SPCE + 1744 582 3 0.423800 2.817273e+00 2.461806e+01 5.932450e+00 # Hw SPCE + 1745 582 4 -0.847600 2.010189e+00 2.426140e+01 6.402994e+00 # Ow SPCE + 1746 582 3 0.423800 1.499435e+00 2.366530e+01 5.783477e+00 # Hw SPCE + 1747 583 3 0.423800 1.234255e+01 1.396176e+01 2.037507e+01 # Hw SPCE + 1748 583 4 -0.847600 1.153188e+01 1.453863e+01 2.027487e+01 # Ow SPCE + 1749 583 3 0.423800 1.071536e+01 1.396252e+01 2.023761e+01 # Hw SPCE + 1750 584 3 0.423800 2.810909e+00 2.179238e+01 7.310922e+00 # Hw SPCE + 1751 584 4 -0.847600 3.683406e+00 2.228090e+01 7.320812e+00 # Ow SPCE + 1752 584 3 0.423800 4.405822e+00 2.166653e+01 7.638090e+00 # Hw SPCE + 1753 585 3 0.423800 8.448996e+00 2.068506e+01 1.886024e+00 # Hw SPCE + 1754 585 4 -0.847600 8.120115e+00 2.158805e+01 1.609529e+00 # Ow SPCE + 1755 585 3 0.423800 8.825993e+00 2.227138e+01 1.796065e+00 # Hw SPCE + 1756 586 3 0.423800 1.552035e+01 1.189869e+01 1.283899e+01 # Hw SPCE + 1757 586 4 -0.847600 1.524184e+01 1.098045e+01 1.255745e+01 # Ow SPCE + 1758 586 3 0.423800 1.602690e+01 1.036326e+01 1.260993e+01 # Hw SPCE + 1759 587 3 0.423800 1.037404e+01 1.680250e+01 1.410571e+01 # Hw SPCE + 1760 587 4 -0.847600 9.390321e+00 1.671053e+01 1.395132e+01 # Ow SPCE + 1761 587 3 0.423800 9.132024e+00 1.574640e+01 1.401253e+01 # Hw SPCE + 1762 588 3 0.423800 1.925143e+01 2.771036e+01 1.136823e+01 # Hw SPCE + 1763 588 4 -0.847600 1.825753e+01 2.762221e+01 1.143454e+01 # Ow SPCE + 1764 588 3 0.423800 1.797827e+01 2.770904e+01 1.239082e+01 # Hw SPCE + 1765 589 3 0.423800 5.368954e+00 1.319885e+01 2.246010e+01 # Hw SPCE + 1766 589 4 -0.847600 5.623630e+00 1.391457e+01 2.311039e+01 # Ow SPCE + 1767 589 3 0.423800 4.872485e+00 1.456903e+01 2.319682e+01 # Hw SPCE + 1768 590 3 0.423800 2.502067e+01 2.248026e+01 2.658176e+01 # Hw SPCE + 1769 590 4 -0.847600 2.503552e+01 2.164180e+01 2.712652e+01 # Ow SPCE + 1770 590 3 0.423800 2.494862e+01 2.084997e+01 2.652199e+01 # Hw SPCE + 1771 591 3 0.423800 2.771210e+01 9.857496e+00 2.349471e+01 # Hw SPCE + 1772 591 4 -0.847600 2.758172e+01 1.077731e+01 2.312465e+01 # Ow SPCE + 1773 591 3 0.423800 2.770545e+01 1.075806e+01 2.213252e+01 # Hw SPCE + 1774 592 3 0.423800 2.236208e+01 1.289022e+01 1.568659e+01 # Hw SPCE + 1775 592 4 -0.847600 2.242230e+01 1.193201e+01 1.596625e+01 # Ow SPCE + 1776 592 3 0.423800 2.300802e+01 1.185651e+01 1.677324e+01 # Hw SPCE + 1777 593 3 0.423800 1.988201e+01 1.279194e+01 1.687017e+01 # Hw SPCE + 1778 593 4 -0.847600 2.021378e+01 1.300179e+01 1.778990e+01 # Ow SPCE + 1779 593 3 0.423800 2.066551e+01 1.219616e+01 1.817317e+01 # Hw SPCE + 1780 594 3 0.423800 1.468952e+01 2.770673e+01 5.255360e+00 # Hw SPCE + 1781 594 4 -0.847600 1.548366e+01 2.712680e+01 5.437112e+00 # Ow SPCE + 1782 594 3 0.423800 1.628119e+01 2.770158e+01 5.620343e+00 # Hw SPCE + 1783 595 3 0.423800 8.562539e+00 2.043599e+01 4.622579e+00 # Hw SPCE + 1784 595 4 -0.847600 8.510816e+00 2.140691e+01 4.856324e+00 # Ow SPCE + 1785 595 3 0.423800 7.721353e+00 2.181791e+01 4.400437e+00 # Hw SPCE + 1786 596 3 0.423800 2.770600e+01 1.600690e+01 2.686389e+01 # Hw SPCE + 1787 596 4 -0.847600 2.714850e+01 1.666860e+01 2.736525e+01 # Ow SPCE + 1788 596 3 0.423800 2.770517e+01 1.746675e+01 2.759565e+01 # Hw SPCE + 1789 597 3 0.423800 1.659849e+01 2.708928e+01 1.606821e+01 # Hw SPCE + 1790 597 4 -0.847600 1.561098e+01 2.693173e+01 1.606789e+01 # Ow SPCE + 1791 597 3 0.423800 1.515025e+01 2.770484e+01 1.563196e+01 # Hw SPCE + 1792 598 3 0.423800 2.769724e+01 2.030543e+01 2.142814e+01 # Hw SPCE + 1793 598 4 -0.847600 2.694008e+01 2.095316e+01 2.151277e+01 # Ow SPCE + 1794 598 3 0.423800 2.730269e+01 2.188246e+01 2.158290e+01 # Hw SPCE + 1795 599 3 0.423800 8.916687e+00 5.895426e+00 1.772221e+01 # Hw SPCE + 1796 599 4 -0.847600 9.042735e+00 5.158155e+00 1.705847e+01 # Ow SPCE + 1797 599 3 0.423800 9.690399e+00 5.450297e+00 1.635478e+01 # Hw SPCE + 1798 600 3 0.423800 1.753578e+01 1.707234e+01 2.389138e+01 # Hw SPCE + 1799 600 4 -0.847600 1.803190e+01 1.621348e+01 2.401877e+01 # Ow SPCE + 1800 600 3 0.423800 1.745017e+01 1.544840e+01 2.374267e+01 # Hw SPCE + +Bonds + + 1 1 2 1 # Ow-Hw + 2 1 3 2 # Ow-Hw + 3 1 5 4 # Ow-Hw + 4 1 6 5 # Ow-Hw + 5 1 8 7 # Ow-Hw + 6 1 9 8 # Ow-Hw + 7 1 11 10 # Ow-Hw + 8 1 12 11 # Ow-Hw + 9 1 14 13 # Ow-Hw + 10 1 15 14 # Ow-Hw + 11 1 17 16 # Ow-Hw + 12 1 18 17 # Ow-Hw + 13 1 20 19 # Ow-Hw + 14 1 21 20 # Ow-Hw + 15 1 23 22 # Ow-Hw + 16 1 24 23 # Ow-Hw + 17 1 26 25 # Ow-Hw + 18 1 27 26 # Ow-Hw + 19 1 29 28 # Ow-Hw + 20 1 30 29 # Ow-Hw + 21 1 32 31 # Ow-Hw + 22 1 33 32 # Ow-Hw + 23 1 35 34 # Ow-Hw + 24 1 36 35 # Ow-Hw + 25 1 38 37 # Ow-Hw + 26 1 39 38 # Ow-Hw + 27 1 41 40 # Ow-Hw + 28 1 42 41 # Ow-Hw + 29 1 44 43 # Ow-Hw + 30 1 45 44 # Ow-Hw + 31 1 47 46 # Ow-Hw + 32 1 48 47 # Ow-Hw + 33 1 50 49 # Ow-Hw + 34 1 51 50 # Ow-Hw + 35 1 53 52 # Ow-Hw + 36 1 54 53 # Ow-Hw + 37 1 56 55 # Ow-Hw + 38 1 57 56 # Ow-Hw + 39 1 59 58 # Ow-Hw + 40 1 60 59 # Ow-Hw + 41 1 62 61 # Ow-Hw + 42 1 63 62 # Ow-Hw + 43 1 65 64 # Ow-Hw + 44 1 66 65 # Ow-Hw + 45 1 68 67 # Ow-Hw + 46 1 69 68 # Ow-Hw + 47 1 71 70 # Ow-Hw + 48 1 72 71 # Ow-Hw + 49 1 74 73 # Ow-Hw + 50 1 75 74 # Ow-Hw + 51 1 77 76 # Ow-Hw + 52 1 78 77 # Ow-Hw + 53 1 80 79 # Ow-Hw + 54 1 81 80 # Ow-Hw + 55 1 83 82 # Ow-Hw + 56 1 84 83 # Ow-Hw + 57 1 86 85 # Ow-Hw + 58 1 87 86 # Ow-Hw + 59 1 89 88 # Ow-Hw + 60 1 90 89 # Ow-Hw + 61 1 92 91 # Ow-Hw + 62 1 93 92 # Ow-Hw + 63 1 95 94 # Ow-Hw + 64 1 96 95 # Ow-Hw + 65 1 98 97 # Ow-Hw + 66 1 99 98 # Ow-Hw + 67 1 101 100 # Ow-Hw + 68 1 102 101 # Ow-Hw + 69 1 104 103 # Ow-Hw + 70 1 105 104 # Ow-Hw + 71 1 107 106 # Ow-Hw + 72 1 108 107 # Ow-Hw + 73 1 110 109 # Ow-Hw + 74 1 111 110 # Ow-Hw + 75 1 113 112 # Ow-Hw + 76 1 114 113 # Ow-Hw + 77 1 116 115 # Ow-Hw + 78 1 117 116 # Ow-Hw + 79 1 119 118 # Ow-Hw + 80 1 120 119 # Ow-Hw + 81 1 122 121 # Ow-Hw + 82 1 123 122 # Ow-Hw + 83 1 125 124 # Ow-Hw + 84 1 126 125 # Ow-Hw + 85 1 128 127 # Ow-Hw + 86 1 129 128 # Ow-Hw + 87 1 131 130 # Ow-Hw + 88 1 132 131 # Ow-Hw + 89 1 134 133 # Ow-Hw + 90 1 135 134 # Ow-Hw + 91 1 137 136 # Ow-Hw + 92 1 138 137 # Ow-Hw + 93 1 140 139 # Ow-Hw + 94 1 141 140 # Ow-Hw + 95 1 143 142 # Ow-Hw + 96 1 144 143 # Ow-Hw + 97 1 146 145 # Ow-Hw + 98 1 147 146 # Ow-Hw + 99 1 149 148 # Ow-Hw + 100 1 150 149 # Ow-Hw + 101 1 152 151 # Ow-Hw + 102 1 153 152 # Ow-Hw + 103 1 155 154 # Ow-Hw + 104 1 156 155 # Ow-Hw + 105 1 158 157 # Ow-Hw + 106 1 159 158 # Ow-Hw + 107 1 161 160 # Ow-Hw + 108 1 162 161 # Ow-Hw + 109 1 164 163 # Ow-Hw + 110 1 165 164 # Ow-Hw + 111 1 167 166 # Ow-Hw + 112 1 168 167 # Ow-Hw + 113 1 170 169 # Ow-Hw + 114 1 171 170 # Ow-Hw + 115 1 173 172 # Ow-Hw + 116 1 174 173 # Ow-Hw + 117 1 176 175 # Ow-Hw + 118 1 177 176 # Ow-Hw + 119 1 179 178 # Ow-Hw + 120 1 180 179 # Ow-Hw + 121 1 182 181 # Ow-Hw + 122 1 183 182 # Ow-Hw + 123 1 185 184 # Ow-Hw + 124 1 186 185 # Ow-Hw + 125 1 188 187 # Ow-Hw + 126 1 189 188 # Ow-Hw + 127 1 191 190 # Ow-Hw + 128 1 192 191 # Ow-Hw + 129 1 194 193 # Ow-Hw + 130 1 195 194 # Ow-Hw + 131 1 197 196 # Ow-Hw + 132 1 198 197 # Ow-Hw + 133 1 200 199 # Ow-Hw + 134 1 201 200 # Ow-Hw + 135 1 203 202 # Ow-Hw + 136 1 204 203 # Ow-Hw + 137 1 206 205 # Ow-Hw + 138 1 207 206 # Ow-Hw + 139 1 209 208 # Ow-Hw + 140 1 210 209 # Ow-Hw + 141 1 212 211 # Ow-Hw + 142 1 213 212 # Ow-Hw + 143 1 215 214 # Ow-Hw + 144 1 216 215 # Ow-Hw + 145 1 218 217 # Ow-Hw + 146 1 219 218 # Ow-Hw + 147 1 221 220 # Ow-Hw + 148 1 222 221 # Ow-Hw + 149 1 224 223 # Ow-Hw + 150 1 225 224 # Ow-Hw + 151 1 227 226 # Ow-Hw + 152 1 228 227 # Ow-Hw + 153 1 230 229 # Ow-Hw + 154 1 231 230 # Ow-Hw + 155 1 233 232 # Ow-Hw + 156 1 234 233 # Ow-Hw + 157 1 236 235 # Ow-Hw + 158 1 237 236 # Ow-Hw + 159 1 239 238 # Ow-Hw + 160 1 240 239 # Ow-Hw + 161 1 242 241 # Ow-Hw + 162 1 243 242 # Ow-Hw + 163 1 245 244 # Ow-Hw + 164 1 246 245 # Ow-Hw + 165 1 248 247 # Ow-Hw + 166 1 249 248 # Ow-Hw + 167 1 251 250 # Ow-Hw + 168 1 252 251 # Ow-Hw + 169 1 254 253 # Ow-Hw + 170 1 255 254 # Ow-Hw + 171 1 257 256 # Ow-Hw + 172 1 258 257 # Ow-Hw + 173 1 260 259 # Ow-Hw + 174 1 261 260 # Ow-Hw + 175 1 263 262 # Ow-Hw + 176 1 264 263 # Ow-Hw + 177 1 266 265 # Ow-Hw + 178 1 267 266 # Ow-Hw + 179 1 269 268 # Ow-Hw + 180 1 270 269 # Ow-Hw + 181 1 272 271 # Ow-Hw + 182 1 273 272 # Ow-Hw + 183 1 275 274 # Ow-Hw + 184 1 276 275 # Ow-Hw + 185 1 278 277 # Ow-Hw + 186 1 279 278 # Ow-Hw + 187 1 281 280 # Ow-Hw + 188 1 282 281 # Ow-Hw + 189 1 284 283 # Ow-Hw + 190 1 285 284 # Ow-Hw + 191 1 287 286 # Ow-Hw + 192 1 288 287 # Ow-Hw + 193 1 290 289 # Ow-Hw + 194 1 291 290 # Ow-Hw + 195 1 293 292 # Ow-Hw + 196 1 294 293 # Ow-Hw + 197 1 296 295 # Ow-Hw + 198 1 297 296 # Ow-Hw + 199 1 299 298 # Ow-Hw + 200 1 300 299 # Ow-Hw + 201 1 302 301 # Ow-Hw + 202 1 303 302 # Ow-Hw + 203 1 305 304 # Ow-Hw + 204 1 306 305 # Ow-Hw + 205 1 308 307 # Ow-Hw + 206 1 309 308 # Ow-Hw + 207 1 311 310 # Ow-Hw + 208 1 312 311 # Ow-Hw + 209 1 314 313 # Ow-Hw + 210 1 315 314 # Ow-Hw + 211 1 317 316 # Ow-Hw + 212 1 318 317 # Ow-Hw + 213 1 320 319 # Ow-Hw + 214 1 321 320 # Ow-Hw + 215 1 323 322 # Ow-Hw + 216 1 324 323 # Ow-Hw + 217 1 326 325 # Ow-Hw + 218 1 327 326 # Ow-Hw + 219 1 329 328 # Ow-Hw + 220 1 330 329 # Ow-Hw + 221 1 332 331 # Ow-Hw + 222 1 333 332 # Ow-Hw + 223 1 335 334 # Ow-Hw + 224 1 336 335 # Ow-Hw + 225 1 338 337 # Ow-Hw + 226 1 339 338 # Ow-Hw + 227 1 341 340 # Ow-Hw + 228 1 342 341 # Ow-Hw + 229 1 344 343 # Ow-Hw + 230 1 345 344 # Ow-Hw + 231 1 347 346 # Ow-Hw + 232 1 348 347 # Ow-Hw + 233 1 350 349 # Ow-Hw + 234 1 351 350 # Ow-Hw + 235 1 353 352 # Ow-Hw + 236 1 354 353 # Ow-Hw + 237 1 356 355 # Ow-Hw + 238 1 357 356 # Ow-Hw + 239 1 359 358 # Ow-Hw + 240 1 360 359 # Ow-Hw + 241 1 362 361 # Ow-Hw + 242 1 363 362 # Ow-Hw + 243 1 365 364 # Ow-Hw + 244 1 366 365 # Ow-Hw + 245 1 368 367 # Ow-Hw + 246 1 369 368 # Ow-Hw + 247 1 371 370 # Ow-Hw + 248 1 372 371 # Ow-Hw + 249 1 374 373 # Ow-Hw + 250 1 375 374 # Ow-Hw + 251 1 377 376 # Ow-Hw + 252 1 378 377 # Ow-Hw + 253 1 380 379 # Ow-Hw + 254 1 381 380 # Ow-Hw + 255 1 383 382 # Ow-Hw + 256 1 384 383 # Ow-Hw + 257 1 386 385 # Ow-Hw + 258 1 387 386 # Ow-Hw + 259 1 389 388 # Ow-Hw + 260 1 390 389 # Ow-Hw + 261 1 392 391 # Ow-Hw + 262 1 393 392 # Ow-Hw + 263 1 395 394 # Ow-Hw + 264 1 396 395 # Ow-Hw + 265 1 398 397 # Ow-Hw + 266 1 399 398 # Ow-Hw + 267 1 401 400 # Ow-Hw + 268 1 402 401 # Ow-Hw + 269 1 404 403 # Ow-Hw + 270 1 405 404 # Ow-Hw + 271 1 407 406 # Ow-Hw + 272 1 408 407 # Ow-Hw + 273 1 410 409 # Ow-Hw + 274 1 411 410 # Ow-Hw + 275 1 413 412 # Ow-Hw + 276 1 414 413 # Ow-Hw + 277 1 416 415 # Ow-Hw + 278 1 417 416 # Ow-Hw + 279 1 419 418 # Ow-Hw + 280 1 420 419 # Ow-Hw + 281 1 422 421 # Ow-Hw + 282 1 423 422 # Ow-Hw + 283 1 425 424 # Ow-Hw + 284 1 426 425 # Ow-Hw + 285 1 428 427 # Ow-Hw + 286 1 429 428 # Ow-Hw + 287 1 431 430 # Ow-Hw + 288 1 432 431 # Ow-Hw + 289 1 434 433 # Ow-Hw + 290 1 435 434 # Ow-Hw + 291 1 437 436 # Ow-Hw + 292 1 438 437 # Ow-Hw + 293 1 440 439 # Ow-Hw + 294 1 441 440 # Ow-Hw + 295 1 443 442 # Ow-Hw + 296 1 444 443 # Ow-Hw + 297 1 446 445 # Ow-Hw + 298 1 447 446 # Ow-Hw + 299 1 449 448 # Ow-Hw + 300 1 450 449 # Ow-Hw + 301 1 452 451 # Ow-Hw + 302 1 453 452 # Ow-Hw + 303 1 455 454 # Ow-Hw + 304 1 456 455 # Ow-Hw + 305 1 458 457 # Ow-Hw + 306 1 459 458 # Ow-Hw + 307 1 461 460 # Ow-Hw + 308 1 462 461 # Ow-Hw + 309 1 464 463 # Ow-Hw + 310 1 465 464 # Ow-Hw + 311 1 467 466 # Ow-Hw + 312 1 468 467 # Ow-Hw + 313 1 470 469 # Ow-Hw + 314 1 471 470 # Ow-Hw + 315 1 473 472 # Ow-Hw + 316 1 474 473 # Ow-Hw + 317 1 476 475 # Ow-Hw + 318 1 477 476 # Ow-Hw + 319 1 479 478 # Ow-Hw + 320 1 480 479 # Ow-Hw + 321 1 482 481 # Ow-Hw + 322 1 483 482 # Ow-Hw + 323 1 485 484 # Ow-Hw + 324 1 486 485 # Ow-Hw + 325 1 488 487 # Ow-Hw + 326 1 489 488 # Ow-Hw + 327 1 491 490 # Ow-Hw + 328 1 492 491 # Ow-Hw + 329 1 494 493 # Ow-Hw + 330 1 495 494 # Ow-Hw + 331 1 497 496 # Ow-Hw + 332 1 498 497 # Ow-Hw + 333 1 500 499 # Ow-Hw + 334 1 501 500 # Ow-Hw + 335 1 503 502 # Ow-Hw + 336 1 504 503 # Ow-Hw + 337 1 506 505 # Ow-Hw + 338 1 507 506 # Ow-Hw + 339 1 509 508 # Ow-Hw + 340 1 510 509 # Ow-Hw + 341 1 512 511 # Ow-Hw + 342 1 513 512 # Ow-Hw + 343 1 515 514 # Ow-Hw + 344 1 516 515 # Ow-Hw + 345 1 518 517 # Ow-Hw + 346 1 519 518 # Ow-Hw + 347 1 521 520 # Ow-Hw + 348 1 522 521 # Ow-Hw + 349 1 524 523 # Ow-Hw + 350 1 525 524 # Ow-Hw + 351 1 527 526 # Ow-Hw + 352 1 528 527 # Ow-Hw + 353 1 530 529 # Ow-Hw + 354 1 531 530 # Ow-Hw + 355 1 533 532 # Ow-Hw + 356 1 534 533 # Ow-Hw + 357 1 536 535 # Ow-Hw + 358 1 537 536 # Ow-Hw + 359 1 539 538 # Ow-Hw + 360 1 540 539 # Ow-Hw + 361 1 542 541 # Ow-Hw + 362 1 543 542 # Ow-Hw + 363 1 545 544 # Ow-Hw + 364 1 546 545 # Ow-Hw + 365 1 548 547 # Ow-Hw + 366 1 549 548 # Ow-Hw + 367 1 551 550 # Ow-Hw + 368 1 552 551 # Ow-Hw + 369 1 554 553 # Ow-Hw + 370 1 555 554 # Ow-Hw + 371 1 557 556 # Ow-Hw + 372 1 558 557 # Ow-Hw + 373 1 560 559 # Ow-Hw + 374 1 561 560 # Ow-Hw + 375 1 563 562 # Ow-Hw + 376 1 564 563 # Ow-Hw + 377 1 566 565 # Ow-Hw + 378 1 567 566 # Ow-Hw + 379 1 569 568 # Ow-Hw + 380 1 570 569 # Ow-Hw + 381 1 572 571 # Ow-Hw + 382 1 573 572 # Ow-Hw + 383 1 575 574 # Ow-Hw + 384 1 576 575 # Ow-Hw + 385 1 578 577 # Ow-Hw + 386 1 579 578 # Ow-Hw + 387 1 581 580 # Ow-Hw + 388 1 582 581 # Ow-Hw + 389 1 584 583 # Ow-Hw + 390 1 585 584 # Ow-Hw + 391 1 587 586 # Ow-Hw + 392 1 588 587 # Ow-Hw + 393 1 590 589 # Ow-Hw + 394 1 591 590 # Ow-Hw + 395 1 593 592 # Ow-Hw + 396 1 594 593 # Ow-Hw + 397 1 596 595 # Ow-Hw + 398 1 597 596 # Ow-Hw + 399 1 599 598 # Ow-Hw + 400 1 600 599 # Ow-Hw + 401 1 602 601 # Ow-Hw + 402 1 603 602 # Ow-Hw + 403 1 605 604 # Ow-Hw + 404 1 606 605 # Ow-Hw + 405 1 608 607 # Ow-Hw + 406 1 609 608 # Ow-Hw + 407 1 611 610 # Ow-Hw + 408 1 612 611 # Ow-Hw + 409 1 614 613 # Ow-Hw + 410 1 615 614 # Ow-Hw + 411 1 617 616 # Ow-Hw + 412 1 618 617 # Ow-Hw + 413 1 620 619 # Ow-Hw + 414 1 621 620 # Ow-Hw + 415 1 623 622 # Ow-Hw + 416 1 624 623 # Ow-Hw + 417 1 626 625 # Ow-Hw + 418 1 627 626 # Ow-Hw + 419 1 629 628 # Ow-Hw + 420 1 630 629 # Ow-Hw + 421 1 632 631 # Ow-Hw + 422 1 633 632 # Ow-Hw + 423 1 635 634 # Ow-Hw + 424 1 636 635 # Ow-Hw + 425 1 638 637 # Ow-Hw + 426 1 639 638 # Ow-Hw + 427 1 641 640 # Ow-Hw + 428 1 642 641 # Ow-Hw + 429 1 644 643 # Ow-Hw + 430 1 645 644 # Ow-Hw + 431 1 647 646 # Ow-Hw + 432 1 648 647 # Ow-Hw + 433 1 650 649 # Ow-Hw + 434 1 651 650 # Ow-Hw + 435 1 653 652 # Ow-Hw + 436 1 654 653 # Ow-Hw + 437 1 656 655 # Ow-Hw + 438 1 657 656 # Ow-Hw + 439 1 659 658 # Ow-Hw + 440 1 660 659 # Ow-Hw + 441 1 662 661 # Ow-Hw + 442 1 663 662 # Ow-Hw + 443 1 665 664 # Ow-Hw + 444 1 666 665 # Ow-Hw + 445 1 668 667 # Ow-Hw + 446 1 669 668 # Ow-Hw + 447 1 671 670 # Ow-Hw + 448 1 672 671 # Ow-Hw + 449 1 674 673 # Ow-Hw + 450 1 675 674 # Ow-Hw + 451 1 677 676 # Ow-Hw + 452 1 678 677 # Ow-Hw + 453 1 680 679 # Ow-Hw + 454 1 681 680 # Ow-Hw + 455 1 683 682 # Ow-Hw + 456 1 684 683 # Ow-Hw + 457 1 686 685 # Ow-Hw + 458 1 687 686 # Ow-Hw + 459 1 689 688 # Ow-Hw + 460 1 690 689 # Ow-Hw + 461 1 692 691 # Ow-Hw + 462 1 693 692 # Ow-Hw + 463 1 695 694 # Ow-Hw + 464 1 696 695 # Ow-Hw + 465 1 698 697 # Ow-Hw + 466 1 699 698 # Ow-Hw + 467 1 701 700 # Ow-Hw + 468 1 702 701 # Ow-Hw + 469 1 704 703 # Ow-Hw + 470 1 705 704 # Ow-Hw + 471 1 707 706 # Ow-Hw + 472 1 708 707 # Ow-Hw + 473 1 710 709 # Ow-Hw + 474 1 711 710 # Ow-Hw + 475 1 713 712 # Ow-Hw + 476 1 714 713 # Ow-Hw + 477 1 716 715 # Ow-Hw + 478 1 717 716 # Ow-Hw + 479 1 719 718 # Ow-Hw + 480 1 720 719 # Ow-Hw + 481 1 722 721 # Ow-Hw + 482 1 723 722 # Ow-Hw + 483 1 725 724 # Ow-Hw + 484 1 726 725 # Ow-Hw + 485 1 728 727 # Ow-Hw + 486 1 729 728 # Ow-Hw + 487 1 731 730 # Ow-Hw + 488 1 732 731 # Ow-Hw + 489 1 734 733 # Ow-Hw + 490 1 735 734 # Ow-Hw + 491 1 737 736 # Ow-Hw + 492 1 738 737 # Ow-Hw + 493 1 740 739 # Ow-Hw + 494 1 741 740 # Ow-Hw + 495 1 743 742 # Ow-Hw + 496 1 744 743 # Ow-Hw + 497 1 746 745 # Ow-Hw + 498 1 747 746 # Ow-Hw + 499 1 749 748 # Ow-Hw + 500 1 750 749 # Ow-Hw + 501 1 752 751 # Ow-Hw + 502 1 753 752 # Ow-Hw + 503 1 755 754 # Ow-Hw + 504 1 756 755 # Ow-Hw + 505 1 758 757 # Ow-Hw + 506 1 759 758 # Ow-Hw + 507 1 761 760 # Ow-Hw + 508 1 762 761 # Ow-Hw + 509 1 764 763 # Ow-Hw + 510 1 765 764 # Ow-Hw + 511 1 767 766 # Ow-Hw + 512 1 768 767 # Ow-Hw + 513 1 770 769 # Ow-Hw + 514 1 771 770 # Ow-Hw + 515 1 773 772 # Ow-Hw + 516 1 774 773 # Ow-Hw + 517 1 776 775 # Ow-Hw + 518 1 777 776 # Ow-Hw + 519 1 779 778 # Ow-Hw + 520 1 780 779 # Ow-Hw + 521 1 782 781 # Ow-Hw + 522 1 783 782 # Ow-Hw + 523 1 785 784 # Ow-Hw + 524 1 786 785 # Ow-Hw + 525 1 788 787 # Ow-Hw + 526 1 789 788 # Ow-Hw + 527 1 791 790 # Ow-Hw + 528 1 792 791 # Ow-Hw + 529 1 794 793 # Ow-Hw + 530 1 795 794 # Ow-Hw + 531 1 797 796 # Ow-Hw + 532 1 798 797 # Ow-Hw + 533 1 800 799 # Ow-Hw + 534 1 801 800 # Ow-Hw + 535 1 803 802 # Ow-Hw + 536 1 804 803 # Ow-Hw + 537 1 806 805 # Ow-Hw + 538 1 807 806 # Ow-Hw + 539 1 809 808 # Ow-Hw + 540 1 810 809 # Ow-Hw + 541 1 812 811 # Ow-Hw + 542 1 813 812 # Ow-Hw + 543 1 815 814 # Ow-Hw + 544 1 816 815 # Ow-Hw + 545 1 818 817 # Ow-Hw + 546 1 819 818 # Ow-Hw + 547 1 821 820 # Ow-Hw + 548 1 822 821 # Ow-Hw + 549 1 824 823 # Ow-Hw + 550 1 825 824 # Ow-Hw + 551 1 827 826 # Ow-Hw + 552 1 828 827 # Ow-Hw + 553 1 830 829 # Ow-Hw + 554 1 831 830 # Ow-Hw + 555 1 833 832 # Ow-Hw + 556 1 834 833 # Ow-Hw + 557 1 836 835 # Ow-Hw + 558 1 837 836 # Ow-Hw + 559 1 839 838 # Ow-Hw + 560 1 840 839 # Ow-Hw + 561 1 842 841 # Ow-Hw + 562 1 843 842 # Ow-Hw + 563 1 845 844 # Ow-Hw + 564 1 846 845 # Ow-Hw + 565 1 848 847 # Ow-Hw + 566 1 849 848 # Ow-Hw + 567 1 851 850 # Ow-Hw + 568 1 852 851 # Ow-Hw + 569 1 854 853 # Ow-Hw + 570 1 855 854 # Ow-Hw + 571 1 857 856 # Ow-Hw + 572 1 858 857 # Ow-Hw + 573 1 860 859 # Ow-Hw + 574 1 861 860 # Ow-Hw + 575 1 863 862 # Ow-Hw + 576 1 864 863 # Ow-Hw + 577 1 866 865 # Ow-Hw + 578 1 867 866 # Ow-Hw + 579 1 869 868 # Ow-Hw + 580 1 870 869 # Ow-Hw + 581 1 872 871 # Ow-Hw + 582 1 873 872 # Ow-Hw + 583 1 875 874 # Ow-Hw + 584 1 876 875 # Ow-Hw + 585 1 878 877 # Ow-Hw + 586 1 879 878 # Ow-Hw + 587 1 881 880 # Ow-Hw + 588 1 882 881 # Ow-Hw + 589 1 884 883 # Ow-Hw + 590 1 885 884 # Ow-Hw + 591 1 887 886 # Ow-Hw + 592 1 888 887 # Ow-Hw + 593 1 890 889 # Ow-Hw + 594 1 891 890 # Ow-Hw + 595 1 893 892 # Ow-Hw + 596 1 894 893 # Ow-Hw + 597 1 896 895 # Ow-Hw + 598 1 897 896 # Ow-Hw + 599 1 899 898 # Ow-Hw + 600 1 900 899 # Ow-Hw + 601 1 902 901 # Ow-Hw + 602 1 903 902 # Ow-Hw + 603 1 905 904 # Ow-Hw + 604 1 906 905 # Ow-Hw + 605 1 908 907 # Ow-Hw + 606 1 909 908 # Ow-Hw + 607 1 911 910 # Ow-Hw + 608 1 912 911 # Ow-Hw + 609 1 914 913 # Ow-Hw + 610 1 915 914 # Ow-Hw + 611 1 917 916 # Ow-Hw + 612 1 918 917 # Ow-Hw + 613 1 920 919 # Ow-Hw + 614 1 921 920 # Ow-Hw + 615 1 923 922 # Ow-Hw + 616 1 924 923 # Ow-Hw + 617 1 926 925 # Ow-Hw + 618 1 927 926 # Ow-Hw + 619 1 929 928 # Ow-Hw + 620 1 930 929 # Ow-Hw + 621 1 932 931 # Ow-Hw + 622 1 933 932 # Ow-Hw + 623 1 935 934 # Ow-Hw + 624 1 936 935 # Ow-Hw + 625 1 938 937 # Ow-Hw + 626 1 939 938 # Ow-Hw + 627 1 941 940 # Ow-Hw + 628 1 942 941 # Ow-Hw + 629 1 944 943 # Ow-Hw + 630 1 945 944 # Ow-Hw + 631 1 947 946 # Ow-Hw + 632 1 948 947 # Ow-Hw + 633 1 950 949 # Ow-Hw + 634 1 951 950 # Ow-Hw + 635 1 953 952 # Ow-Hw + 636 1 954 953 # Ow-Hw + 637 1 956 955 # Ow-Hw + 638 1 957 956 # Ow-Hw + 639 1 959 958 # Ow-Hw + 640 1 960 959 # Ow-Hw + 641 1 962 961 # Ow-Hw + 642 1 963 962 # Ow-Hw + 643 1 965 964 # Ow-Hw + 644 1 966 965 # Ow-Hw + 645 1 968 967 # Ow-Hw + 646 1 969 968 # Ow-Hw + 647 1 971 970 # Ow-Hw + 648 1 972 971 # Ow-Hw + 649 1 974 973 # Ow-Hw + 650 1 975 974 # Ow-Hw + 651 1 977 976 # Ow-Hw + 652 1 978 977 # Ow-Hw + 653 1 980 979 # Ow-Hw + 654 1 981 980 # Ow-Hw + 655 1 983 982 # Ow-Hw + 656 1 984 983 # Ow-Hw + 657 1 986 985 # Ow-Hw + 658 1 987 986 # Ow-Hw + 659 1 989 988 # Ow-Hw + 660 1 990 989 # Ow-Hw + 661 1 992 991 # Ow-Hw + 662 1 993 992 # Ow-Hw + 663 1 995 994 # Ow-Hw + 664 1 996 995 # Ow-Hw + 665 1 998 997 # Ow-Hw + 666 1 999 998 # Ow-Hw + 667 1 1001 1000 # Ow-Hw + 668 1 1002 1001 # Ow-Hw + 669 1 1004 1003 # Ow-Hw + 670 1 1005 1004 # Ow-Hw + 671 1 1007 1006 # Ow-Hw + 672 1 1008 1007 # Ow-Hw + 673 1 1010 1009 # Ow-Hw + 674 1 1011 1010 # Ow-Hw + 675 1 1013 1012 # Ow-Hw + 676 1 1014 1013 # Ow-Hw + 677 1 1016 1015 # Ow-Hw + 678 1 1017 1016 # Ow-Hw + 679 1 1019 1018 # Ow-Hw + 680 1 1020 1019 # Ow-Hw + 681 1 1022 1021 # Ow-Hw + 682 1 1023 1022 # Ow-Hw + 683 1 1025 1024 # Ow-Hw + 684 1 1026 1025 # Ow-Hw + 685 1 1028 1027 # Ow-Hw + 686 1 1029 1028 # Ow-Hw + 687 1 1031 1030 # Ow-Hw + 688 1 1032 1031 # Ow-Hw + 689 1 1034 1033 # Ow-Hw + 690 1 1035 1034 # Ow-Hw + 691 1 1037 1036 # Ow-Hw + 692 1 1038 1037 # Ow-Hw + 693 1 1040 1039 # Ow-Hw + 694 1 1041 1040 # Ow-Hw + 695 1 1043 1042 # Ow-Hw + 696 1 1044 1043 # Ow-Hw + 697 1 1046 1045 # Ow-Hw + 698 1 1047 1046 # Ow-Hw + 699 1 1049 1048 # Ow-Hw + 700 1 1050 1049 # Ow-Hw + 701 1 1052 1051 # Ow-Hw + 702 1 1053 1052 # Ow-Hw + 703 1 1055 1054 # Ow-Hw + 704 1 1056 1055 # Ow-Hw + 705 1 1058 1057 # Ow-Hw + 706 1 1059 1058 # Ow-Hw + 707 1 1061 1060 # Ow-Hw + 708 1 1062 1061 # Ow-Hw + 709 1 1064 1063 # Ow-Hw + 710 1 1065 1064 # Ow-Hw + 711 1 1067 1066 # Ow-Hw + 712 1 1068 1067 # Ow-Hw + 713 1 1070 1069 # Ow-Hw + 714 1 1071 1070 # Ow-Hw + 715 1 1073 1072 # Ow-Hw + 716 1 1074 1073 # Ow-Hw + 717 1 1076 1075 # Ow-Hw + 718 1 1077 1076 # Ow-Hw + 719 1 1079 1078 # Ow-Hw + 720 1 1080 1079 # Ow-Hw + 721 1 1082 1081 # Ow-Hw + 722 1 1083 1082 # Ow-Hw + 723 1 1085 1084 # Ow-Hw + 724 1 1086 1085 # Ow-Hw + 725 1 1088 1087 # Ow-Hw + 726 1 1089 1088 # Ow-Hw + 727 1 1091 1090 # Ow-Hw + 728 1 1092 1091 # Ow-Hw + 729 1 1094 1093 # Ow-Hw + 730 1 1095 1094 # Ow-Hw + 731 1 1097 1096 # Ow-Hw + 732 1 1098 1097 # Ow-Hw + 733 1 1100 1099 # Ow-Hw + 734 1 1101 1100 # Ow-Hw + 735 1 1103 1102 # Ow-Hw + 736 1 1104 1103 # Ow-Hw + 737 1 1106 1105 # Ow-Hw + 738 1 1107 1106 # Ow-Hw + 739 1 1109 1108 # Ow-Hw + 740 1 1110 1109 # Ow-Hw + 741 1 1112 1111 # Ow-Hw + 742 1 1113 1112 # Ow-Hw + 743 1 1115 1114 # Ow-Hw + 744 1 1116 1115 # Ow-Hw + 745 1 1118 1117 # Ow-Hw + 746 1 1119 1118 # Ow-Hw + 747 1 1121 1120 # Ow-Hw + 748 1 1122 1121 # Ow-Hw + 749 1 1124 1123 # Ow-Hw + 750 1 1125 1124 # Ow-Hw + 751 1 1127 1126 # Ow-Hw + 752 1 1128 1127 # Ow-Hw + 753 1 1130 1129 # Ow-Hw + 754 1 1131 1130 # Ow-Hw + 755 1 1133 1132 # Ow-Hw + 756 1 1134 1133 # Ow-Hw + 757 1 1136 1135 # Ow-Hw + 758 1 1137 1136 # Ow-Hw + 759 1 1139 1138 # Ow-Hw + 760 1 1140 1139 # Ow-Hw + 761 1 1142 1141 # Ow-Hw + 762 1 1143 1142 # Ow-Hw + 763 1 1145 1144 # Ow-Hw + 764 1 1146 1145 # Ow-Hw + 765 1 1148 1147 # Ow-Hw + 766 1 1149 1148 # Ow-Hw + 767 1 1151 1150 # Ow-Hw + 768 1 1152 1151 # Ow-Hw + 769 1 1154 1153 # Ow-Hw + 770 1 1155 1154 # Ow-Hw + 771 1 1157 1156 # Ow-Hw + 772 1 1158 1157 # Ow-Hw + 773 1 1160 1159 # Ow-Hw + 774 1 1161 1160 # Ow-Hw + 775 1 1163 1162 # Ow-Hw + 776 1 1164 1163 # Ow-Hw + 777 1 1166 1165 # Ow-Hw + 778 1 1167 1166 # Ow-Hw + 779 1 1169 1168 # Ow-Hw + 780 1 1170 1169 # Ow-Hw + 781 1 1172 1171 # Ow-Hw + 782 1 1173 1172 # Ow-Hw + 783 1 1175 1174 # Ow-Hw + 784 1 1176 1175 # Ow-Hw + 785 1 1178 1177 # Ow-Hw + 786 1 1179 1178 # Ow-Hw + 787 1 1181 1180 # Ow-Hw + 788 1 1182 1181 # Ow-Hw + 789 1 1184 1183 # Ow-Hw + 790 1 1185 1184 # Ow-Hw + 791 1 1187 1186 # Ow-Hw + 792 1 1188 1187 # Ow-Hw + 793 1 1190 1189 # Ow-Hw + 794 1 1191 1190 # Ow-Hw + 795 1 1193 1192 # Ow-Hw + 796 1 1194 1193 # Ow-Hw + 797 1 1196 1195 # Ow-Hw + 798 1 1197 1196 # Ow-Hw + 799 1 1199 1198 # Ow-Hw + 800 1 1200 1199 # Ow-Hw + 801 1 1202 1201 # Ow-Hw + 802 1 1203 1202 # Ow-Hw + 803 1 1205 1204 # Ow-Hw + 804 1 1206 1205 # Ow-Hw + 805 1 1208 1207 # Ow-Hw + 806 1 1209 1208 # Ow-Hw + 807 1 1211 1210 # Ow-Hw + 808 1 1212 1211 # Ow-Hw + 809 1 1214 1213 # Ow-Hw + 810 1 1215 1214 # Ow-Hw + 811 1 1217 1216 # Ow-Hw + 812 1 1218 1217 # Ow-Hw + 813 1 1220 1219 # Ow-Hw + 814 1 1221 1220 # Ow-Hw + 815 1 1223 1222 # Ow-Hw + 816 1 1224 1223 # Ow-Hw + 817 1 1226 1225 # Ow-Hw + 818 1 1227 1226 # Ow-Hw + 819 1 1229 1228 # Ow-Hw + 820 1 1230 1229 # Ow-Hw + 821 1 1232 1231 # Ow-Hw + 822 1 1233 1232 # Ow-Hw + 823 1 1235 1234 # Ow-Hw + 824 1 1236 1235 # Ow-Hw + 825 1 1238 1237 # Ow-Hw + 826 1 1239 1238 # Ow-Hw + 827 1 1241 1240 # Ow-Hw + 828 1 1242 1241 # Ow-Hw + 829 1 1244 1243 # Ow-Hw + 830 1 1245 1244 # Ow-Hw + 831 1 1247 1246 # Ow-Hw + 832 1 1248 1247 # Ow-Hw + 833 1 1250 1249 # Ow-Hw + 834 1 1251 1250 # Ow-Hw + 835 1 1253 1252 # Ow-Hw + 836 1 1254 1253 # Ow-Hw + 837 1 1256 1255 # Ow-Hw + 838 1 1257 1256 # Ow-Hw + 839 1 1259 1258 # Ow-Hw + 840 1 1260 1259 # Ow-Hw + 841 1 1262 1261 # Ow-Hw + 842 1 1263 1262 # Ow-Hw + 843 1 1265 1264 # Ow-Hw + 844 1 1266 1265 # Ow-Hw + 845 1 1268 1267 # Ow-Hw + 846 1 1269 1268 # Ow-Hw + 847 1 1271 1270 # Ow-Hw + 848 1 1272 1271 # Ow-Hw + 849 1 1274 1273 # Ow-Hw + 850 1 1275 1274 # Ow-Hw + 851 1 1277 1276 # Ow-Hw + 852 1 1278 1277 # Ow-Hw + 853 1 1280 1279 # Ow-Hw + 854 1 1281 1280 # Ow-Hw + 855 1 1283 1282 # Ow-Hw + 856 1 1284 1283 # Ow-Hw + 857 1 1286 1285 # Ow-Hw + 858 1 1287 1286 # Ow-Hw + 859 1 1289 1288 # Ow-Hw + 860 1 1290 1289 # Ow-Hw + 861 1 1292 1291 # Ow-Hw + 862 1 1293 1292 # Ow-Hw + 863 1 1295 1294 # Ow-Hw + 864 1 1296 1295 # Ow-Hw + 865 1 1298 1297 # Ow-Hw + 866 1 1299 1298 # Ow-Hw + 867 1 1301 1300 # Ow-Hw + 868 1 1302 1301 # Ow-Hw + 869 1 1304 1303 # Ow-Hw + 870 1 1305 1304 # Ow-Hw + 871 1 1307 1306 # Ow-Hw + 872 1 1308 1307 # Ow-Hw + 873 1 1310 1309 # Ow-Hw + 874 1 1311 1310 # Ow-Hw + 875 1 1313 1312 # Ow-Hw + 876 1 1314 1313 # Ow-Hw + 877 1 1316 1315 # Ow-Hw + 878 1 1317 1316 # Ow-Hw + 879 1 1319 1318 # Ow-Hw + 880 1 1320 1319 # Ow-Hw + 881 1 1322 1321 # Ow-Hw + 882 1 1323 1322 # Ow-Hw + 883 1 1325 1324 # Ow-Hw + 884 1 1326 1325 # Ow-Hw + 885 1 1328 1327 # Ow-Hw + 886 1 1329 1328 # Ow-Hw + 887 1 1331 1330 # Ow-Hw + 888 1 1332 1331 # Ow-Hw + 889 1 1334 1333 # Ow-Hw + 890 1 1335 1334 # Ow-Hw + 891 1 1337 1336 # Ow-Hw + 892 1 1338 1337 # Ow-Hw + 893 1 1340 1339 # Ow-Hw + 894 1 1341 1340 # Ow-Hw + 895 1 1343 1342 # Ow-Hw + 896 1 1344 1343 # Ow-Hw + 897 1 1346 1345 # Ow-Hw + 898 1 1347 1346 # Ow-Hw + 899 1 1349 1348 # Ow-Hw + 900 1 1350 1349 # Ow-Hw + 901 1 1352 1351 # Ow-Hw + 902 1 1353 1352 # Ow-Hw + 903 1 1355 1354 # Ow-Hw + 904 1 1356 1355 # Ow-Hw + 905 1 1358 1357 # Ow-Hw + 906 1 1359 1358 # Ow-Hw + 907 1 1361 1360 # Ow-Hw + 908 1 1362 1361 # Ow-Hw + 909 1 1364 1363 # Ow-Hw + 910 1 1365 1364 # Ow-Hw + 911 1 1367 1366 # Ow-Hw + 912 1 1368 1367 # Ow-Hw + 913 1 1370 1369 # Ow-Hw + 914 1 1371 1370 # Ow-Hw + 915 1 1373 1372 # Ow-Hw + 916 1 1374 1373 # Ow-Hw + 917 1 1376 1375 # Ow-Hw + 918 1 1377 1376 # Ow-Hw + 919 1 1379 1378 # Ow-Hw + 920 1 1380 1379 # Ow-Hw + 921 1 1382 1381 # Ow-Hw + 922 1 1383 1382 # Ow-Hw + 923 1 1385 1384 # Ow-Hw + 924 1 1386 1385 # Ow-Hw + 925 1 1388 1387 # Ow-Hw + 926 1 1389 1388 # Ow-Hw + 927 1 1391 1390 # Ow-Hw + 928 1 1392 1391 # Ow-Hw + 929 1 1394 1393 # Ow-Hw + 930 1 1395 1394 # Ow-Hw + 931 1 1397 1396 # Ow-Hw + 932 1 1398 1397 # Ow-Hw + 933 1 1400 1399 # Ow-Hw + 934 1 1401 1400 # Ow-Hw + 935 1 1403 1402 # Ow-Hw + 936 1 1404 1403 # Ow-Hw + 937 1 1406 1405 # Ow-Hw + 938 1 1407 1406 # Ow-Hw + 939 1 1409 1408 # Ow-Hw + 940 1 1410 1409 # Ow-Hw + 941 1 1412 1411 # Ow-Hw + 942 1 1413 1412 # Ow-Hw + 943 1 1415 1414 # Ow-Hw + 944 1 1416 1415 # Ow-Hw + 945 1 1418 1417 # Ow-Hw + 946 1 1419 1418 # Ow-Hw + 947 1 1421 1420 # Ow-Hw + 948 1 1422 1421 # Ow-Hw + 949 1 1424 1423 # Ow-Hw + 950 1 1425 1424 # Ow-Hw + 951 1 1427 1426 # Ow-Hw + 952 1 1428 1427 # Ow-Hw + 953 1 1430 1429 # Ow-Hw + 954 1 1431 1430 # Ow-Hw + 955 1 1433 1432 # Ow-Hw + 956 1 1434 1433 # Ow-Hw + 957 1 1436 1435 # Ow-Hw + 958 1 1437 1436 # Ow-Hw + 959 1 1439 1438 # Ow-Hw + 960 1 1440 1439 # Ow-Hw + 961 1 1442 1441 # Ow-Hw + 962 1 1443 1442 # Ow-Hw + 963 1 1445 1444 # Ow-Hw + 964 1 1446 1445 # Ow-Hw + 965 1 1448 1447 # Ow-Hw + 966 1 1449 1448 # Ow-Hw + 967 1 1451 1450 # Ow-Hw + 968 1 1452 1451 # Ow-Hw + 969 1 1454 1453 # Ow-Hw + 970 1 1455 1454 # Ow-Hw + 971 1 1457 1456 # Ow-Hw + 972 1 1458 1457 # Ow-Hw + 973 1 1460 1459 # Ow-Hw + 974 1 1461 1460 # Ow-Hw + 975 1 1463 1462 # Ow-Hw + 976 1 1464 1463 # Ow-Hw + 977 1 1466 1465 # Ow-Hw + 978 1 1467 1466 # Ow-Hw + 979 1 1469 1468 # Ow-Hw + 980 1 1470 1469 # Ow-Hw + 981 1 1472 1471 # Ow-Hw + 982 1 1473 1472 # Ow-Hw + 983 1 1475 1474 # Ow-Hw + 984 1 1476 1475 # Ow-Hw + 985 1 1478 1477 # Ow-Hw + 986 1 1479 1478 # Ow-Hw + 987 1 1481 1480 # Ow-Hw + 988 1 1482 1481 # Ow-Hw + 989 1 1484 1483 # Ow-Hw + 990 1 1485 1484 # Ow-Hw + 991 1 1487 1486 # Ow-Hw + 992 1 1488 1487 # Ow-Hw + 993 1 1490 1489 # Ow-Hw + 994 1 1491 1490 # Ow-Hw + 995 1 1493 1492 # Ow-Hw + 996 1 1494 1493 # Ow-Hw + 997 1 1496 1495 # Ow-Hw + 998 1 1497 1496 # Ow-Hw + 999 1 1499 1498 # Ow-Hw + 1000 1 1500 1499 # Ow-Hw + 1001 1 1502 1501 # Ow-Hw + 1002 1 1503 1502 # Ow-Hw + 1003 1 1505 1504 # Ow-Hw + 1004 1 1506 1505 # Ow-Hw + 1005 1 1508 1507 # Ow-Hw + 1006 1 1509 1508 # Ow-Hw + 1007 1 1511 1510 # Ow-Hw + 1008 1 1512 1511 # Ow-Hw + 1009 1 1514 1513 # Ow-Hw + 1010 1 1515 1514 # Ow-Hw + 1011 1 1517 1516 # Ow-Hw + 1012 1 1518 1517 # Ow-Hw + 1013 1 1520 1519 # Ow-Hw + 1014 1 1521 1520 # Ow-Hw + 1015 1 1523 1522 # Ow-Hw + 1016 1 1524 1523 # Ow-Hw + 1017 1 1526 1525 # Ow-Hw + 1018 1 1527 1526 # Ow-Hw + 1019 1 1529 1528 # Ow-Hw + 1020 1 1530 1529 # Ow-Hw + 1021 1 1532 1531 # Ow-Hw + 1022 1 1533 1532 # Ow-Hw + 1023 1 1535 1534 # Ow-Hw + 1024 1 1536 1535 # Ow-Hw + 1025 1 1538 1537 # Ow-Hw + 1026 1 1539 1538 # Ow-Hw + 1027 1 1541 1540 # Ow-Hw + 1028 1 1542 1541 # Ow-Hw + 1029 1 1544 1543 # Ow-Hw + 1030 1 1545 1544 # Ow-Hw + 1031 1 1547 1546 # Ow-Hw + 1032 1 1548 1547 # Ow-Hw + 1033 1 1550 1549 # Ow-Hw + 1034 1 1551 1550 # Ow-Hw + 1035 1 1553 1552 # Ow-Hw + 1036 1 1554 1553 # Ow-Hw + 1037 1 1556 1555 # Ow-Hw + 1038 1 1557 1556 # Ow-Hw + 1039 1 1559 1558 # Ow-Hw + 1040 1 1560 1559 # Ow-Hw + 1041 1 1562 1561 # Ow-Hw + 1042 1 1563 1562 # Ow-Hw + 1043 1 1565 1564 # Ow-Hw + 1044 1 1566 1565 # Ow-Hw + 1045 1 1568 1567 # Ow-Hw + 1046 1 1569 1568 # Ow-Hw + 1047 1 1571 1570 # Ow-Hw + 1048 1 1572 1571 # Ow-Hw + 1049 1 1574 1573 # Ow-Hw + 1050 1 1575 1574 # Ow-Hw + 1051 1 1577 1576 # Ow-Hw + 1052 1 1578 1577 # Ow-Hw + 1053 1 1580 1579 # Ow-Hw + 1054 1 1581 1580 # Ow-Hw + 1055 1 1583 1582 # Ow-Hw + 1056 1 1584 1583 # Ow-Hw + 1057 1 1586 1585 # Ow-Hw + 1058 1 1587 1586 # Ow-Hw + 1059 1 1589 1588 # Ow-Hw + 1060 1 1590 1589 # Ow-Hw + 1061 1 1592 1591 # Ow-Hw + 1062 1 1593 1592 # Ow-Hw + 1063 1 1595 1594 # Ow-Hw + 1064 1 1596 1595 # Ow-Hw + 1065 1 1598 1597 # Ow-Hw + 1066 1 1599 1598 # Ow-Hw + 1067 1 1601 1600 # Ow-Hw + 1068 1 1602 1601 # Ow-Hw + 1069 1 1604 1603 # Ow-Hw + 1070 1 1605 1604 # Ow-Hw + 1071 1 1607 1606 # Ow-Hw + 1072 1 1608 1607 # Ow-Hw + 1073 1 1610 1609 # Ow-Hw + 1074 1 1611 1610 # Ow-Hw + 1075 1 1613 1612 # Ow-Hw + 1076 1 1614 1613 # Ow-Hw + 1077 1 1616 1615 # Ow-Hw + 1078 1 1617 1616 # Ow-Hw + 1079 1 1619 1618 # Ow-Hw + 1080 1 1620 1619 # Ow-Hw + 1081 1 1622 1621 # Ow-Hw + 1082 1 1623 1622 # Ow-Hw + 1083 1 1625 1624 # Ow-Hw + 1084 1 1626 1625 # Ow-Hw + 1085 1 1628 1627 # Ow-Hw + 1086 1 1629 1628 # Ow-Hw + 1087 1 1631 1630 # Ow-Hw + 1088 1 1632 1631 # Ow-Hw + 1089 1 1634 1633 # Ow-Hw + 1090 1 1635 1634 # Ow-Hw + 1091 1 1637 1636 # Ow-Hw + 1092 1 1638 1637 # Ow-Hw + 1093 1 1640 1639 # Ow-Hw + 1094 1 1641 1640 # Ow-Hw + 1095 1 1643 1642 # Ow-Hw + 1096 1 1644 1643 # Ow-Hw + 1097 1 1646 1645 # Ow-Hw + 1098 1 1647 1646 # Ow-Hw + 1099 1 1649 1648 # Ow-Hw + 1100 1 1650 1649 # Ow-Hw + 1101 1 1652 1651 # Ow-Hw + 1102 1 1653 1652 # Ow-Hw + 1103 1 1655 1654 # Ow-Hw + 1104 1 1656 1655 # Ow-Hw + 1105 1 1658 1657 # Ow-Hw + 1106 1 1659 1658 # Ow-Hw + 1107 1 1661 1660 # Ow-Hw + 1108 1 1662 1661 # Ow-Hw + 1109 1 1664 1663 # Ow-Hw + 1110 1 1665 1664 # Ow-Hw + 1111 1 1667 1666 # Ow-Hw + 1112 1 1668 1667 # Ow-Hw + 1113 1 1670 1669 # Ow-Hw + 1114 1 1671 1670 # Ow-Hw + 1115 1 1673 1672 # Ow-Hw + 1116 1 1674 1673 # Ow-Hw + 1117 1 1676 1675 # Ow-Hw + 1118 1 1677 1676 # Ow-Hw + 1119 1 1679 1678 # Ow-Hw + 1120 1 1680 1679 # Ow-Hw + 1121 1 1682 1681 # Ow-Hw + 1122 1 1683 1682 # Ow-Hw + 1123 1 1685 1684 # Ow-Hw + 1124 1 1686 1685 # Ow-Hw + 1125 1 1688 1687 # Ow-Hw + 1126 1 1689 1688 # Ow-Hw + 1127 1 1691 1690 # Ow-Hw + 1128 1 1692 1691 # Ow-Hw + 1129 1 1694 1693 # Ow-Hw + 1130 1 1695 1694 # Ow-Hw + 1131 1 1697 1696 # Ow-Hw + 1132 1 1698 1697 # Ow-Hw + 1133 1 1700 1699 # Ow-Hw + 1134 1 1701 1700 # Ow-Hw + 1135 1 1703 1702 # Ow-Hw + 1136 1 1704 1703 # Ow-Hw + 1137 1 1706 1705 # Ow-Hw + 1138 1 1707 1706 # Ow-Hw + 1139 1 1709 1708 # Ow-Hw + 1140 1 1710 1709 # Ow-Hw + 1141 1 1712 1711 # Ow-Hw + 1142 1 1713 1712 # Ow-Hw + 1143 1 1715 1714 # Ow-Hw + 1144 1 1716 1715 # Ow-Hw + 1145 1 1718 1717 # Ow-Hw + 1146 1 1719 1718 # Ow-Hw + 1147 1 1721 1720 # Ow-Hw + 1148 1 1722 1721 # Ow-Hw + 1149 1 1724 1723 # Ow-Hw + 1150 1 1725 1724 # Ow-Hw + 1151 1 1727 1726 # Ow-Hw + 1152 1 1728 1727 # Ow-Hw + 1153 1 1730 1729 # Ow-Hw + 1154 1 1731 1730 # Ow-Hw + 1155 1 1733 1732 # Ow-Hw + 1156 1 1734 1733 # Ow-Hw + 1157 1 1736 1735 # Ow-Hw + 1158 1 1737 1736 # Ow-Hw + 1159 1 1739 1738 # Ow-Hw + 1160 1 1740 1739 # Ow-Hw + 1161 1 1742 1741 # Ow-Hw + 1162 1 1743 1742 # Ow-Hw + 1163 1 1745 1744 # Ow-Hw + 1164 1 1746 1745 # Ow-Hw + 1165 1 1748 1747 # Ow-Hw + 1166 1 1749 1748 # Ow-Hw + 1167 1 1751 1750 # Ow-Hw + 1168 1 1752 1751 # Ow-Hw + 1169 1 1754 1753 # Ow-Hw + 1170 1 1755 1754 # Ow-Hw + 1171 1 1757 1756 # Ow-Hw + 1172 1 1758 1757 # Ow-Hw + 1173 1 1760 1759 # Ow-Hw + 1174 1 1761 1760 # Ow-Hw + 1175 1 1763 1762 # Ow-Hw + 1176 1 1764 1763 # Ow-Hw + 1177 1 1766 1765 # Ow-Hw + 1178 1 1767 1766 # Ow-Hw + 1179 1 1769 1768 # Ow-Hw + 1180 1 1770 1769 # Ow-Hw + 1181 1 1772 1771 # Ow-Hw + 1182 1 1773 1772 # Ow-Hw + 1183 1 1775 1774 # Ow-Hw + 1184 1 1776 1775 # Ow-Hw + 1185 1 1778 1777 # Ow-Hw + 1186 1 1779 1778 # Ow-Hw + 1187 1 1781 1780 # Ow-Hw + 1188 1 1782 1781 # Ow-Hw + 1189 1 1784 1783 # Ow-Hw + 1190 1 1785 1784 # Ow-Hw + 1191 1 1787 1786 # Ow-Hw + 1192 1 1788 1787 # Ow-Hw + 1193 1 1790 1789 # Ow-Hw + 1194 1 1791 1790 # Ow-Hw + 1195 1 1793 1792 # Ow-Hw + 1196 1 1794 1793 # Ow-Hw + 1197 1 1796 1795 # Ow-Hw + 1198 1 1797 1796 # Ow-Hw + 1199 1 1799 1798 # Ow-Hw + 1200 1 1800 1799 # Ow-Hw + +Angles + + 1 1 1 2 3 # Hw-Ow-Hw + 2 1 4 5 6 # Hw-Ow-Hw + 3 1 7 8 9 # Hw-Ow-Hw + 4 1 10 11 12 # Hw-Ow-Hw + 5 1 13 14 15 # Hw-Ow-Hw + 6 1 16 17 18 # Hw-Ow-Hw + 7 1 19 20 21 # Hw-Ow-Hw + 8 1 22 23 24 # Hw-Ow-Hw + 9 1 25 26 27 # Hw-Ow-Hw + 10 1 28 29 30 # Hw-Ow-Hw + 11 1 31 32 33 # Hw-Ow-Hw + 12 1 34 35 36 # Hw-Ow-Hw + 13 1 37 38 39 # Hw-Ow-Hw + 14 1 40 41 42 # Hw-Ow-Hw + 15 1 43 44 45 # Hw-Ow-Hw + 16 1 46 47 48 # Hw-Ow-Hw + 17 1 49 50 51 # Hw-Ow-Hw + 18 1 52 53 54 # Hw-Ow-Hw + 19 1 55 56 57 # Hw-Ow-Hw + 20 1 58 59 60 # Hw-Ow-Hw + 21 1 61 62 63 # Hw-Ow-Hw + 22 1 64 65 66 # Hw-Ow-Hw + 23 1 67 68 69 # Hw-Ow-Hw + 24 1 70 71 72 # Hw-Ow-Hw + 25 1 73 74 75 # Hw-Ow-Hw + 26 1 76 77 78 # Hw-Ow-Hw + 27 1 79 80 81 # Hw-Ow-Hw + 28 1 82 83 84 # Hw-Ow-Hw + 29 1 85 86 87 # Hw-Ow-Hw + 30 1 88 89 90 # Hw-Ow-Hw + 31 1 91 92 93 # Hw-Ow-Hw + 32 1 94 95 96 # Hw-Ow-Hw + 33 1 97 98 99 # Hw-Ow-Hw + 34 1 100 101 102 # Hw-Ow-Hw + 35 1 103 104 105 # Hw-Ow-Hw + 36 1 106 107 108 # Hw-Ow-Hw + 37 1 109 110 111 # Hw-Ow-Hw + 38 1 112 113 114 # Hw-Ow-Hw + 39 1 115 116 117 # Hw-Ow-Hw + 40 1 118 119 120 # Hw-Ow-Hw + 41 1 121 122 123 # Hw-Ow-Hw + 42 1 124 125 126 # Hw-Ow-Hw + 43 1 127 128 129 # Hw-Ow-Hw + 44 1 130 131 132 # Hw-Ow-Hw + 45 1 133 134 135 # Hw-Ow-Hw + 46 1 136 137 138 # Hw-Ow-Hw + 47 1 139 140 141 # Hw-Ow-Hw + 48 1 142 143 144 # Hw-Ow-Hw + 49 1 145 146 147 # Hw-Ow-Hw + 50 1 148 149 150 # Hw-Ow-Hw + 51 1 151 152 153 # Hw-Ow-Hw + 52 1 154 155 156 # Hw-Ow-Hw + 53 1 157 158 159 # Hw-Ow-Hw + 54 1 160 161 162 # Hw-Ow-Hw + 55 1 163 164 165 # Hw-Ow-Hw + 56 1 166 167 168 # Hw-Ow-Hw + 57 1 169 170 171 # Hw-Ow-Hw + 58 1 172 173 174 # Hw-Ow-Hw + 59 1 175 176 177 # Hw-Ow-Hw + 60 1 178 179 180 # Hw-Ow-Hw + 61 1 181 182 183 # Hw-Ow-Hw + 62 1 184 185 186 # Hw-Ow-Hw + 63 1 187 188 189 # Hw-Ow-Hw + 64 1 190 191 192 # Hw-Ow-Hw + 65 1 193 194 195 # Hw-Ow-Hw + 66 1 196 197 198 # Hw-Ow-Hw + 67 1 199 200 201 # Hw-Ow-Hw + 68 1 202 203 204 # Hw-Ow-Hw + 69 1 205 206 207 # Hw-Ow-Hw + 70 1 208 209 210 # Hw-Ow-Hw + 71 1 211 212 213 # Hw-Ow-Hw + 72 1 214 215 216 # Hw-Ow-Hw + 73 1 217 218 219 # Hw-Ow-Hw + 74 1 220 221 222 # Hw-Ow-Hw + 75 1 223 224 225 # Hw-Ow-Hw + 76 1 226 227 228 # Hw-Ow-Hw + 77 1 229 230 231 # Hw-Ow-Hw + 78 1 232 233 234 # Hw-Ow-Hw + 79 1 235 236 237 # Hw-Ow-Hw + 80 1 238 239 240 # Hw-Ow-Hw + 81 1 241 242 243 # Hw-Ow-Hw + 82 1 244 245 246 # Hw-Ow-Hw + 83 1 247 248 249 # Hw-Ow-Hw + 84 1 250 251 252 # Hw-Ow-Hw + 85 1 253 254 255 # Hw-Ow-Hw + 86 1 256 257 258 # Hw-Ow-Hw + 87 1 259 260 261 # Hw-Ow-Hw + 88 1 262 263 264 # Hw-Ow-Hw + 89 1 265 266 267 # Hw-Ow-Hw + 90 1 268 269 270 # Hw-Ow-Hw + 91 1 271 272 273 # Hw-Ow-Hw + 92 1 274 275 276 # Hw-Ow-Hw + 93 1 277 278 279 # Hw-Ow-Hw + 94 1 280 281 282 # Hw-Ow-Hw + 95 1 283 284 285 # Hw-Ow-Hw + 96 1 286 287 288 # Hw-Ow-Hw + 97 1 289 290 291 # Hw-Ow-Hw + 98 1 292 293 294 # Hw-Ow-Hw + 99 1 295 296 297 # Hw-Ow-Hw + 100 1 298 299 300 # Hw-Ow-Hw + 101 1 301 302 303 # Hw-Ow-Hw + 102 1 304 305 306 # Hw-Ow-Hw + 103 1 307 308 309 # Hw-Ow-Hw + 104 1 310 311 312 # Hw-Ow-Hw + 105 1 313 314 315 # Hw-Ow-Hw + 106 1 316 317 318 # Hw-Ow-Hw + 107 1 319 320 321 # Hw-Ow-Hw + 108 1 322 323 324 # Hw-Ow-Hw + 109 1 325 326 327 # Hw-Ow-Hw + 110 1 328 329 330 # Hw-Ow-Hw + 111 1 331 332 333 # Hw-Ow-Hw + 112 1 334 335 336 # Hw-Ow-Hw + 113 1 337 338 339 # Hw-Ow-Hw + 114 1 340 341 342 # Hw-Ow-Hw + 115 1 343 344 345 # Hw-Ow-Hw + 116 1 346 347 348 # Hw-Ow-Hw + 117 1 349 350 351 # Hw-Ow-Hw + 118 1 352 353 354 # Hw-Ow-Hw + 119 1 355 356 357 # Hw-Ow-Hw + 120 1 358 359 360 # Hw-Ow-Hw + 121 1 361 362 363 # Hw-Ow-Hw + 122 1 364 365 366 # Hw-Ow-Hw + 123 1 367 368 369 # Hw-Ow-Hw + 124 1 370 371 372 # Hw-Ow-Hw + 125 1 373 374 375 # Hw-Ow-Hw + 126 1 376 377 378 # Hw-Ow-Hw + 127 1 379 380 381 # Hw-Ow-Hw + 128 1 382 383 384 # Hw-Ow-Hw + 129 1 385 386 387 # Hw-Ow-Hw + 130 1 388 389 390 # Hw-Ow-Hw + 131 1 391 392 393 # Hw-Ow-Hw + 132 1 394 395 396 # Hw-Ow-Hw + 133 1 397 398 399 # Hw-Ow-Hw + 134 1 400 401 402 # Hw-Ow-Hw + 135 1 403 404 405 # Hw-Ow-Hw + 136 1 406 407 408 # Hw-Ow-Hw + 137 1 409 410 411 # Hw-Ow-Hw + 138 1 412 413 414 # Hw-Ow-Hw + 139 1 415 416 417 # Hw-Ow-Hw + 140 1 418 419 420 # Hw-Ow-Hw + 141 1 421 422 423 # Hw-Ow-Hw + 142 1 424 425 426 # Hw-Ow-Hw + 143 1 427 428 429 # Hw-Ow-Hw + 144 1 430 431 432 # Hw-Ow-Hw + 145 1 433 434 435 # Hw-Ow-Hw + 146 1 436 437 438 # Hw-Ow-Hw + 147 1 439 440 441 # Hw-Ow-Hw + 148 1 442 443 444 # Hw-Ow-Hw + 149 1 445 446 447 # Hw-Ow-Hw + 150 1 448 449 450 # Hw-Ow-Hw + 151 1 451 452 453 # Hw-Ow-Hw + 152 1 454 455 456 # Hw-Ow-Hw + 153 1 457 458 459 # Hw-Ow-Hw + 154 1 460 461 462 # Hw-Ow-Hw + 155 1 463 464 465 # Hw-Ow-Hw + 156 1 466 467 468 # Hw-Ow-Hw + 157 1 469 470 471 # Hw-Ow-Hw + 158 1 472 473 474 # Hw-Ow-Hw + 159 1 475 476 477 # Hw-Ow-Hw + 160 1 478 479 480 # Hw-Ow-Hw + 161 1 481 482 483 # Hw-Ow-Hw + 162 1 484 485 486 # Hw-Ow-Hw + 163 1 487 488 489 # Hw-Ow-Hw + 164 1 490 491 492 # Hw-Ow-Hw + 165 1 493 494 495 # Hw-Ow-Hw + 166 1 496 497 498 # Hw-Ow-Hw + 167 1 499 500 501 # Hw-Ow-Hw + 168 1 502 503 504 # Hw-Ow-Hw + 169 1 505 506 507 # Hw-Ow-Hw + 170 1 508 509 510 # Hw-Ow-Hw + 171 1 511 512 513 # Hw-Ow-Hw + 172 1 514 515 516 # Hw-Ow-Hw + 173 1 517 518 519 # Hw-Ow-Hw + 174 1 520 521 522 # Hw-Ow-Hw + 175 1 523 524 525 # Hw-Ow-Hw + 176 1 526 527 528 # Hw-Ow-Hw + 177 1 529 530 531 # Hw-Ow-Hw + 178 1 532 533 534 # Hw-Ow-Hw + 179 1 535 536 537 # Hw-Ow-Hw + 180 1 538 539 540 # Hw-Ow-Hw + 181 1 541 542 543 # Hw-Ow-Hw + 182 1 544 545 546 # Hw-Ow-Hw + 183 1 547 548 549 # Hw-Ow-Hw + 184 1 550 551 552 # Hw-Ow-Hw + 185 1 553 554 555 # Hw-Ow-Hw + 186 1 556 557 558 # Hw-Ow-Hw + 187 1 559 560 561 # Hw-Ow-Hw + 188 1 562 563 564 # Hw-Ow-Hw + 189 1 565 566 567 # Hw-Ow-Hw + 190 1 568 569 570 # Hw-Ow-Hw + 191 1 571 572 573 # Hw-Ow-Hw + 192 1 574 575 576 # Hw-Ow-Hw + 193 1 577 578 579 # Hw-Ow-Hw + 194 1 580 581 582 # Hw-Ow-Hw + 195 1 583 584 585 # Hw-Ow-Hw + 196 1 586 587 588 # Hw-Ow-Hw + 197 1 589 590 591 # Hw-Ow-Hw + 198 1 592 593 594 # Hw-Ow-Hw + 199 1 595 596 597 # Hw-Ow-Hw + 200 1 598 599 600 # Hw-Ow-Hw + 201 1 601 602 603 # Hw-Ow-Hw + 202 1 604 605 606 # Hw-Ow-Hw + 203 1 607 608 609 # Hw-Ow-Hw + 204 1 610 611 612 # Hw-Ow-Hw + 205 1 613 614 615 # Hw-Ow-Hw + 206 1 616 617 618 # Hw-Ow-Hw + 207 1 619 620 621 # Hw-Ow-Hw + 208 1 622 623 624 # Hw-Ow-Hw + 209 1 625 626 627 # Hw-Ow-Hw + 210 1 628 629 630 # Hw-Ow-Hw + 211 1 631 632 633 # Hw-Ow-Hw + 212 1 634 635 636 # Hw-Ow-Hw + 213 1 637 638 639 # Hw-Ow-Hw + 214 1 640 641 642 # Hw-Ow-Hw + 215 1 643 644 645 # Hw-Ow-Hw + 216 1 646 647 648 # Hw-Ow-Hw + 217 1 649 650 651 # Hw-Ow-Hw + 218 1 652 653 654 # Hw-Ow-Hw + 219 1 655 656 657 # Hw-Ow-Hw + 220 1 658 659 660 # Hw-Ow-Hw + 221 1 661 662 663 # Hw-Ow-Hw + 222 1 664 665 666 # Hw-Ow-Hw + 223 1 667 668 669 # Hw-Ow-Hw + 224 1 670 671 672 # Hw-Ow-Hw + 225 1 673 674 675 # Hw-Ow-Hw + 226 1 676 677 678 # Hw-Ow-Hw + 227 1 679 680 681 # Hw-Ow-Hw + 228 1 682 683 684 # Hw-Ow-Hw + 229 1 685 686 687 # Hw-Ow-Hw + 230 1 688 689 690 # Hw-Ow-Hw + 231 1 691 692 693 # Hw-Ow-Hw + 232 1 694 695 696 # Hw-Ow-Hw + 233 1 697 698 699 # Hw-Ow-Hw + 234 1 700 701 702 # Hw-Ow-Hw + 235 1 703 704 705 # Hw-Ow-Hw + 236 1 706 707 708 # Hw-Ow-Hw + 237 1 709 710 711 # Hw-Ow-Hw + 238 1 712 713 714 # Hw-Ow-Hw + 239 1 715 716 717 # Hw-Ow-Hw + 240 1 718 719 720 # Hw-Ow-Hw + 241 1 721 722 723 # Hw-Ow-Hw + 242 1 724 725 726 # Hw-Ow-Hw + 243 1 727 728 729 # Hw-Ow-Hw + 244 1 730 731 732 # Hw-Ow-Hw + 245 1 733 734 735 # Hw-Ow-Hw + 246 1 736 737 738 # Hw-Ow-Hw + 247 1 739 740 741 # Hw-Ow-Hw + 248 1 742 743 744 # Hw-Ow-Hw + 249 1 745 746 747 # Hw-Ow-Hw + 250 1 748 749 750 # Hw-Ow-Hw + 251 1 751 752 753 # Hw-Ow-Hw + 252 1 754 755 756 # Hw-Ow-Hw + 253 1 757 758 759 # Hw-Ow-Hw + 254 1 760 761 762 # Hw-Ow-Hw + 255 1 763 764 765 # Hw-Ow-Hw + 256 1 766 767 768 # Hw-Ow-Hw + 257 1 769 770 771 # Hw-Ow-Hw + 258 1 772 773 774 # Hw-Ow-Hw + 259 1 775 776 777 # Hw-Ow-Hw + 260 1 778 779 780 # Hw-Ow-Hw + 261 1 781 782 783 # Hw-Ow-Hw + 262 1 784 785 786 # Hw-Ow-Hw + 263 1 787 788 789 # Hw-Ow-Hw + 264 1 790 791 792 # Hw-Ow-Hw + 265 1 793 794 795 # Hw-Ow-Hw + 266 1 796 797 798 # Hw-Ow-Hw + 267 1 799 800 801 # Hw-Ow-Hw + 268 1 802 803 804 # Hw-Ow-Hw + 269 1 805 806 807 # Hw-Ow-Hw + 270 1 808 809 810 # Hw-Ow-Hw + 271 1 811 812 813 # Hw-Ow-Hw + 272 1 814 815 816 # Hw-Ow-Hw + 273 1 817 818 819 # Hw-Ow-Hw + 274 1 820 821 822 # Hw-Ow-Hw + 275 1 823 824 825 # Hw-Ow-Hw + 276 1 826 827 828 # Hw-Ow-Hw + 277 1 829 830 831 # Hw-Ow-Hw + 278 1 832 833 834 # Hw-Ow-Hw + 279 1 835 836 837 # Hw-Ow-Hw + 280 1 838 839 840 # Hw-Ow-Hw + 281 1 841 842 843 # Hw-Ow-Hw + 282 1 844 845 846 # Hw-Ow-Hw + 283 1 847 848 849 # Hw-Ow-Hw + 284 1 850 851 852 # Hw-Ow-Hw + 285 1 853 854 855 # Hw-Ow-Hw + 286 1 856 857 858 # Hw-Ow-Hw + 287 1 859 860 861 # Hw-Ow-Hw + 288 1 862 863 864 # Hw-Ow-Hw + 289 1 865 866 867 # Hw-Ow-Hw + 290 1 868 869 870 # Hw-Ow-Hw + 291 1 871 872 873 # Hw-Ow-Hw + 292 1 874 875 876 # Hw-Ow-Hw + 293 1 877 878 879 # Hw-Ow-Hw + 294 1 880 881 882 # Hw-Ow-Hw + 295 1 883 884 885 # Hw-Ow-Hw + 296 1 886 887 888 # Hw-Ow-Hw + 297 1 889 890 891 # Hw-Ow-Hw + 298 1 892 893 894 # Hw-Ow-Hw + 299 1 895 896 897 # Hw-Ow-Hw + 300 1 898 899 900 # Hw-Ow-Hw + 301 1 901 902 903 # Hw-Ow-Hw + 302 1 904 905 906 # Hw-Ow-Hw + 303 1 907 908 909 # Hw-Ow-Hw + 304 1 910 911 912 # Hw-Ow-Hw + 305 1 913 914 915 # Hw-Ow-Hw + 306 1 916 917 918 # Hw-Ow-Hw + 307 1 919 920 921 # Hw-Ow-Hw + 308 1 922 923 924 # Hw-Ow-Hw + 309 1 925 926 927 # Hw-Ow-Hw + 310 1 928 929 930 # Hw-Ow-Hw + 311 1 931 932 933 # Hw-Ow-Hw + 312 1 934 935 936 # Hw-Ow-Hw + 313 1 937 938 939 # Hw-Ow-Hw + 314 1 940 941 942 # Hw-Ow-Hw + 315 1 943 944 945 # Hw-Ow-Hw + 316 1 946 947 948 # Hw-Ow-Hw + 317 1 949 950 951 # Hw-Ow-Hw + 318 1 952 953 954 # Hw-Ow-Hw + 319 1 955 956 957 # Hw-Ow-Hw + 320 1 958 959 960 # Hw-Ow-Hw + 321 1 961 962 963 # Hw-Ow-Hw + 322 1 964 965 966 # Hw-Ow-Hw + 323 1 967 968 969 # Hw-Ow-Hw + 324 1 970 971 972 # Hw-Ow-Hw + 325 1 973 974 975 # Hw-Ow-Hw + 326 1 976 977 978 # Hw-Ow-Hw + 327 1 979 980 981 # Hw-Ow-Hw + 328 1 982 983 984 # Hw-Ow-Hw + 329 1 985 986 987 # Hw-Ow-Hw + 330 1 988 989 990 # Hw-Ow-Hw + 331 1 991 992 993 # Hw-Ow-Hw + 332 1 994 995 996 # Hw-Ow-Hw + 333 1 997 998 999 # Hw-Ow-Hw + 334 1 1000 1001 1002 # Hw-Ow-Hw + 335 1 1003 1004 1005 # Hw-Ow-Hw + 336 1 1006 1007 1008 # Hw-Ow-Hw + 337 1 1009 1010 1011 # Hw-Ow-Hw + 338 1 1012 1013 1014 # Hw-Ow-Hw + 339 1 1015 1016 1017 # Hw-Ow-Hw + 340 1 1018 1019 1020 # Hw-Ow-Hw + 341 1 1021 1022 1023 # Hw-Ow-Hw + 342 1 1024 1025 1026 # Hw-Ow-Hw + 343 1 1027 1028 1029 # Hw-Ow-Hw + 344 1 1030 1031 1032 # Hw-Ow-Hw + 345 1 1033 1034 1035 # Hw-Ow-Hw + 346 1 1036 1037 1038 # Hw-Ow-Hw + 347 1 1039 1040 1041 # Hw-Ow-Hw + 348 1 1042 1043 1044 # Hw-Ow-Hw + 349 1 1045 1046 1047 # Hw-Ow-Hw + 350 1 1048 1049 1050 # Hw-Ow-Hw + 351 1 1051 1052 1053 # Hw-Ow-Hw + 352 1 1054 1055 1056 # Hw-Ow-Hw + 353 1 1057 1058 1059 # Hw-Ow-Hw + 354 1 1060 1061 1062 # Hw-Ow-Hw + 355 1 1063 1064 1065 # Hw-Ow-Hw + 356 1 1066 1067 1068 # Hw-Ow-Hw + 357 1 1069 1070 1071 # Hw-Ow-Hw + 358 1 1072 1073 1074 # Hw-Ow-Hw + 359 1 1075 1076 1077 # Hw-Ow-Hw + 360 1 1078 1079 1080 # Hw-Ow-Hw + 361 1 1081 1082 1083 # Hw-Ow-Hw + 362 1 1084 1085 1086 # Hw-Ow-Hw + 363 1 1087 1088 1089 # Hw-Ow-Hw + 364 1 1090 1091 1092 # Hw-Ow-Hw + 365 1 1093 1094 1095 # Hw-Ow-Hw + 366 1 1096 1097 1098 # Hw-Ow-Hw + 367 1 1099 1100 1101 # Hw-Ow-Hw + 368 1 1102 1103 1104 # Hw-Ow-Hw + 369 1 1105 1106 1107 # Hw-Ow-Hw + 370 1 1108 1109 1110 # Hw-Ow-Hw + 371 1 1111 1112 1113 # Hw-Ow-Hw + 372 1 1114 1115 1116 # Hw-Ow-Hw + 373 1 1117 1118 1119 # Hw-Ow-Hw + 374 1 1120 1121 1122 # Hw-Ow-Hw + 375 1 1123 1124 1125 # Hw-Ow-Hw + 376 1 1126 1127 1128 # Hw-Ow-Hw + 377 1 1129 1130 1131 # Hw-Ow-Hw + 378 1 1132 1133 1134 # Hw-Ow-Hw + 379 1 1135 1136 1137 # Hw-Ow-Hw + 380 1 1138 1139 1140 # Hw-Ow-Hw + 381 1 1141 1142 1143 # Hw-Ow-Hw + 382 1 1144 1145 1146 # Hw-Ow-Hw + 383 1 1147 1148 1149 # Hw-Ow-Hw + 384 1 1150 1151 1152 # Hw-Ow-Hw + 385 1 1153 1154 1155 # Hw-Ow-Hw + 386 1 1156 1157 1158 # Hw-Ow-Hw + 387 1 1159 1160 1161 # Hw-Ow-Hw + 388 1 1162 1163 1164 # Hw-Ow-Hw + 389 1 1165 1166 1167 # Hw-Ow-Hw + 390 1 1168 1169 1170 # Hw-Ow-Hw + 391 1 1171 1172 1173 # Hw-Ow-Hw + 392 1 1174 1175 1176 # Hw-Ow-Hw + 393 1 1177 1178 1179 # Hw-Ow-Hw + 394 1 1180 1181 1182 # Hw-Ow-Hw + 395 1 1183 1184 1185 # Hw-Ow-Hw + 396 1 1186 1187 1188 # Hw-Ow-Hw + 397 1 1189 1190 1191 # Hw-Ow-Hw + 398 1 1192 1193 1194 # Hw-Ow-Hw + 399 1 1195 1196 1197 # Hw-Ow-Hw + 400 1 1198 1199 1200 # Hw-Ow-Hw + 401 1 1201 1202 1203 # Hw-Ow-Hw + 402 1 1204 1205 1206 # Hw-Ow-Hw + 403 1 1207 1208 1209 # Hw-Ow-Hw + 404 1 1210 1211 1212 # Hw-Ow-Hw + 405 1 1213 1214 1215 # Hw-Ow-Hw + 406 1 1216 1217 1218 # Hw-Ow-Hw + 407 1 1219 1220 1221 # Hw-Ow-Hw + 408 1 1222 1223 1224 # Hw-Ow-Hw + 409 1 1225 1226 1227 # Hw-Ow-Hw + 410 1 1228 1229 1230 # Hw-Ow-Hw + 411 1 1231 1232 1233 # Hw-Ow-Hw + 412 1 1234 1235 1236 # Hw-Ow-Hw + 413 1 1237 1238 1239 # Hw-Ow-Hw + 414 1 1240 1241 1242 # Hw-Ow-Hw + 415 1 1243 1244 1245 # Hw-Ow-Hw + 416 1 1246 1247 1248 # Hw-Ow-Hw + 417 1 1249 1250 1251 # Hw-Ow-Hw + 418 1 1252 1253 1254 # Hw-Ow-Hw + 419 1 1255 1256 1257 # Hw-Ow-Hw + 420 1 1258 1259 1260 # Hw-Ow-Hw + 421 1 1261 1262 1263 # Hw-Ow-Hw + 422 1 1264 1265 1266 # Hw-Ow-Hw + 423 1 1267 1268 1269 # Hw-Ow-Hw + 424 1 1270 1271 1272 # Hw-Ow-Hw + 425 1 1273 1274 1275 # Hw-Ow-Hw + 426 1 1276 1277 1278 # Hw-Ow-Hw + 427 1 1279 1280 1281 # Hw-Ow-Hw + 428 1 1282 1283 1284 # Hw-Ow-Hw + 429 1 1285 1286 1287 # Hw-Ow-Hw + 430 1 1288 1289 1290 # Hw-Ow-Hw + 431 1 1291 1292 1293 # Hw-Ow-Hw + 432 1 1294 1295 1296 # Hw-Ow-Hw + 433 1 1297 1298 1299 # Hw-Ow-Hw + 434 1 1300 1301 1302 # Hw-Ow-Hw + 435 1 1303 1304 1305 # Hw-Ow-Hw + 436 1 1306 1307 1308 # Hw-Ow-Hw + 437 1 1309 1310 1311 # Hw-Ow-Hw + 438 1 1312 1313 1314 # Hw-Ow-Hw + 439 1 1315 1316 1317 # Hw-Ow-Hw + 440 1 1318 1319 1320 # Hw-Ow-Hw + 441 1 1321 1322 1323 # Hw-Ow-Hw + 442 1 1324 1325 1326 # Hw-Ow-Hw + 443 1 1327 1328 1329 # Hw-Ow-Hw + 444 1 1330 1331 1332 # Hw-Ow-Hw + 445 1 1333 1334 1335 # Hw-Ow-Hw + 446 1 1336 1337 1338 # Hw-Ow-Hw + 447 1 1339 1340 1341 # Hw-Ow-Hw + 448 1 1342 1343 1344 # Hw-Ow-Hw + 449 1 1345 1346 1347 # Hw-Ow-Hw + 450 1 1348 1349 1350 # Hw-Ow-Hw + 451 1 1351 1352 1353 # Hw-Ow-Hw + 452 1 1354 1355 1356 # Hw-Ow-Hw + 453 1 1357 1358 1359 # Hw-Ow-Hw + 454 1 1360 1361 1362 # Hw-Ow-Hw + 455 1 1363 1364 1365 # Hw-Ow-Hw + 456 1 1366 1367 1368 # Hw-Ow-Hw + 457 1 1369 1370 1371 # Hw-Ow-Hw + 458 1 1372 1373 1374 # Hw-Ow-Hw + 459 1 1375 1376 1377 # Hw-Ow-Hw + 460 1 1378 1379 1380 # Hw-Ow-Hw + 461 1 1381 1382 1383 # Hw-Ow-Hw + 462 1 1384 1385 1386 # Hw-Ow-Hw + 463 1 1387 1388 1389 # Hw-Ow-Hw + 464 1 1390 1391 1392 # Hw-Ow-Hw + 465 1 1393 1394 1395 # Hw-Ow-Hw + 466 1 1396 1397 1398 # Hw-Ow-Hw + 467 1 1399 1400 1401 # Hw-Ow-Hw + 468 1 1402 1403 1404 # Hw-Ow-Hw + 469 1 1405 1406 1407 # Hw-Ow-Hw + 470 1 1408 1409 1410 # Hw-Ow-Hw + 471 1 1411 1412 1413 # Hw-Ow-Hw + 472 1 1414 1415 1416 # Hw-Ow-Hw + 473 1 1417 1418 1419 # Hw-Ow-Hw + 474 1 1420 1421 1422 # Hw-Ow-Hw + 475 1 1423 1424 1425 # Hw-Ow-Hw + 476 1 1426 1427 1428 # Hw-Ow-Hw + 477 1 1429 1430 1431 # Hw-Ow-Hw + 478 1 1432 1433 1434 # Hw-Ow-Hw + 479 1 1435 1436 1437 # Hw-Ow-Hw + 480 1 1438 1439 1440 # Hw-Ow-Hw + 481 1 1441 1442 1443 # Hw-Ow-Hw + 482 1 1444 1445 1446 # Hw-Ow-Hw + 483 1 1447 1448 1449 # Hw-Ow-Hw + 484 1 1450 1451 1452 # Hw-Ow-Hw + 485 1 1453 1454 1455 # Hw-Ow-Hw + 486 1 1456 1457 1458 # Hw-Ow-Hw + 487 1 1459 1460 1461 # Hw-Ow-Hw + 488 1 1462 1463 1464 # Hw-Ow-Hw + 489 1 1465 1466 1467 # Hw-Ow-Hw + 490 1 1468 1469 1470 # Hw-Ow-Hw + 491 1 1471 1472 1473 # Hw-Ow-Hw + 492 1 1474 1475 1476 # Hw-Ow-Hw + 493 1 1477 1478 1479 # Hw-Ow-Hw + 494 1 1480 1481 1482 # Hw-Ow-Hw + 495 1 1483 1484 1485 # Hw-Ow-Hw + 496 1 1486 1487 1488 # Hw-Ow-Hw + 497 1 1489 1490 1491 # Hw-Ow-Hw + 498 1 1492 1493 1494 # Hw-Ow-Hw + 499 1 1495 1496 1497 # Hw-Ow-Hw + 500 1 1498 1499 1500 # Hw-Ow-Hw + 501 1 1501 1502 1503 # Hw-Ow-Hw + 502 1 1504 1505 1506 # Hw-Ow-Hw + 503 1 1507 1508 1509 # Hw-Ow-Hw + 504 1 1510 1511 1512 # Hw-Ow-Hw + 505 1 1513 1514 1515 # Hw-Ow-Hw + 506 1 1516 1517 1518 # Hw-Ow-Hw + 507 1 1519 1520 1521 # Hw-Ow-Hw + 508 1 1522 1523 1524 # Hw-Ow-Hw + 509 1 1525 1526 1527 # Hw-Ow-Hw + 510 1 1528 1529 1530 # Hw-Ow-Hw + 511 1 1531 1532 1533 # Hw-Ow-Hw + 512 1 1534 1535 1536 # Hw-Ow-Hw + 513 1 1537 1538 1539 # Hw-Ow-Hw + 514 1 1540 1541 1542 # Hw-Ow-Hw + 515 1 1543 1544 1545 # Hw-Ow-Hw + 516 1 1546 1547 1548 # Hw-Ow-Hw + 517 1 1549 1550 1551 # Hw-Ow-Hw + 518 1 1552 1553 1554 # Hw-Ow-Hw + 519 1 1555 1556 1557 # Hw-Ow-Hw + 520 1 1558 1559 1560 # Hw-Ow-Hw + 521 1 1561 1562 1563 # Hw-Ow-Hw + 522 1 1564 1565 1566 # Hw-Ow-Hw + 523 1 1567 1568 1569 # Hw-Ow-Hw + 524 1 1570 1571 1572 # Hw-Ow-Hw + 525 1 1573 1574 1575 # Hw-Ow-Hw + 526 1 1576 1577 1578 # Hw-Ow-Hw + 527 1 1579 1580 1581 # Hw-Ow-Hw + 528 1 1582 1583 1584 # Hw-Ow-Hw + 529 1 1585 1586 1587 # Hw-Ow-Hw + 530 1 1588 1589 1590 # Hw-Ow-Hw + 531 1 1591 1592 1593 # Hw-Ow-Hw + 532 1 1594 1595 1596 # Hw-Ow-Hw + 533 1 1597 1598 1599 # Hw-Ow-Hw + 534 1 1600 1601 1602 # Hw-Ow-Hw + 535 1 1603 1604 1605 # Hw-Ow-Hw + 536 1 1606 1607 1608 # Hw-Ow-Hw + 537 1 1609 1610 1611 # Hw-Ow-Hw + 538 1 1612 1613 1614 # Hw-Ow-Hw + 539 1 1615 1616 1617 # Hw-Ow-Hw + 540 1 1618 1619 1620 # Hw-Ow-Hw + 541 1 1621 1622 1623 # Hw-Ow-Hw + 542 1 1624 1625 1626 # Hw-Ow-Hw + 543 1 1627 1628 1629 # Hw-Ow-Hw + 544 1 1630 1631 1632 # Hw-Ow-Hw + 545 1 1633 1634 1635 # Hw-Ow-Hw + 546 1 1636 1637 1638 # Hw-Ow-Hw + 547 1 1639 1640 1641 # Hw-Ow-Hw + 548 1 1642 1643 1644 # Hw-Ow-Hw + 549 1 1645 1646 1647 # Hw-Ow-Hw + 550 1 1648 1649 1650 # Hw-Ow-Hw + 551 1 1651 1652 1653 # Hw-Ow-Hw + 552 1 1654 1655 1656 # Hw-Ow-Hw + 553 1 1657 1658 1659 # Hw-Ow-Hw + 554 1 1660 1661 1662 # Hw-Ow-Hw + 555 1 1663 1664 1665 # Hw-Ow-Hw + 556 1 1666 1667 1668 # Hw-Ow-Hw + 557 1 1669 1670 1671 # Hw-Ow-Hw + 558 1 1672 1673 1674 # Hw-Ow-Hw + 559 1 1675 1676 1677 # Hw-Ow-Hw + 560 1 1678 1679 1680 # Hw-Ow-Hw + 561 1 1681 1682 1683 # Hw-Ow-Hw + 562 1 1684 1685 1686 # Hw-Ow-Hw + 563 1 1687 1688 1689 # Hw-Ow-Hw + 564 1 1690 1691 1692 # Hw-Ow-Hw + 565 1 1693 1694 1695 # Hw-Ow-Hw + 566 1 1696 1697 1698 # Hw-Ow-Hw + 567 1 1699 1700 1701 # Hw-Ow-Hw + 568 1 1702 1703 1704 # Hw-Ow-Hw + 569 1 1705 1706 1707 # Hw-Ow-Hw + 570 1 1708 1709 1710 # Hw-Ow-Hw + 571 1 1711 1712 1713 # Hw-Ow-Hw + 572 1 1714 1715 1716 # Hw-Ow-Hw + 573 1 1717 1718 1719 # Hw-Ow-Hw + 574 1 1720 1721 1722 # Hw-Ow-Hw + 575 1 1723 1724 1725 # Hw-Ow-Hw + 576 1 1726 1727 1728 # Hw-Ow-Hw + 577 1 1729 1730 1731 # Hw-Ow-Hw + 578 1 1732 1733 1734 # Hw-Ow-Hw + 579 1 1735 1736 1737 # Hw-Ow-Hw + 580 1 1738 1739 1740 # Hw-Ow-Hw + 581 1 1741 1742 1743 # Hw-Ow-Hw + 582 1 1744 1745 1746 # Hw-Ow-Hw + 583 1 1747 1748 1749 # Hw-Ow-Hw + 584 1 1750 1751 1752 # Hw-Ow-Hw + 585 1 1753 1754 1755 # Hw-Ow-Hw + 586 1 1756 1757 1758 # Hw-Ow-Hw + 587 1 1759 1760 1761 # Hw-Ow-Hw + 588 1 1762 1763 1764 # Hw-Ow-Hw + 589 1 1765 1766 1767 # Hw-Ow-Hw + 590 1 1768 1769 1770 # Hw-Ow-Hw + 591 1 1771 1772 1773 # Hw-Ow-Hw + 592 1 1774 1775 1776 # Hw-Ow-Hw + 593 1 1777 1778 1779 # Hw-Ow-Hw + 594 1 1780 1781 1782 # Hw-Ow-Hw + 595 1 1783 1784 1785 # Hw-Ow-Hw + 596 1 1786 1787 1788 # Hw-Ow-Hw + 597 1 1789 1790 1791 # Hw-Ow-Hw + 598 1 1792 1793 1794 # Hw-Ow-Hw + 599 1 1795 1796 1797 # Hw-Ow-Hw + 600 1 1798 1799 1800 # Hw-Ow-Hw diff --git a/examples/USER/fep/SPCEhyd/mols/spce.ff b/examples/USER/fep/SPCEhyd/mols/spce.ff new file mode 100644 index 0000000000..b9fd69f4fd --- /dev/null +++ b/examples/USER/fep/SPCEhyd/mols/spce.ff @@ -0,0 +1,14 @@ +# SPC/E water (kJ/mol, A, deg) + +ATOMS +# type m/uma q/e pot pars +Ow Ow 15.999 -0.8476 lj 3.1655 0.6503 +Hw Hw 1.008 0.4238 lj 0.0000 0.0000 + +BONDS +# i j pot re/A ka/kJmol-1 +Ow Hw cons 1.000 4331.53 + +ANGLES +# i j k pot th/deg ka/kjmol-1 +Hw Ow Hw harm 109.47 317.57 diff --git a/examples/USER/fep/SPCEhyd/mols/spce.zmat b/examples/USER/fep/SPCEhyd/mols/spce.zmat new file mode 100644 index 0000000000..e5e60c4f1b --- /dev/null +++ b/examples/USER/fep/SPCEhyd/mols/spce.zmat @@ -0,0 +1,7 @@ +SPCE + +Hw +Ow 1 1.000 +Hw 2 1.000 1 109.47 + +spce.ff diff --git a/examples/USER/fep/SPCEhyd/mols/spce_h.ff b/examples/USER/fep/SPCEhyd/mols/spce_h.ff new file mode 100644 index 0000000000..a1c6933404 --- /dev/null +++ b/examples/USER/fep/SPCEhyd/mols/spce_h.ff @@ -0,0 +1,14 @@ +# SPC/E water (kJ/mol, A, deg) + +ATOMS +# type m/uma q/e pot pars +Owh Ow 15.999 -0.8476 lj 3.1655 0.6503 +Hwh Hw 1.008 0.4238 lj 0.0000 0.0000 + +BONDS +# i j pot re/A ka/kJmol-1 +Ow Hw cons 1.000 4331.53 + +ANGLES +# i j k pot th/deg ka/kjmol-1 +Hw Ow Hw harm 109.47 317.57 diff --git a/examples/USER/fep/SPCEhyd/mols/spce_h.zmat b/examples/USER/fep/SPCEhyd/mols/spce_h.zmat new file mode 100644 index 0000000000..54c4cf9107 --- /dev/null +++ b/examples/USER/fep/SPCEhyd/mols/spce_h.zmat @@ -0,0 +1,7 @@ +SPCE + +Hwh +Owh 1 1.000 +Hwh 2 1.000 1 109.47 + +spce_h.ff diff --git a/examples/USER/mdi/README b/examples/USER/mdi/README new file mode 100644 index 0000000000..086341f785 --- /dev/null +++ b/examples/USER/mdi/README @@ -0,0 +1,24 @@ +This dir contains scripts that demonstrate how to use LAMMPS as an +MDI engine. LAMMPS as an engine performs the MD timestepping. +The driver is a simple Python script. Every timestep the driver +sends one or more commands to LAMMPS. + +-------------- + +The Script.sh file has comands to perform some very simple example +runs. + +-------------- + +More complex calculations using LAMMPS as an MDI engine will +typically require the use of an MDI driver. Several MDI drivers +support calculations with LAMMPS, and include: + +Ab Initio Molecular Dynamics (AIMD) Driver: +https://github.com/MolSSI-MDI/MDI_AIMD_Driver + +Nudged Elastic Band (NEB) Driver: +https://github.com/MolSSI-MDI/MDI_NEB_Driver + +Metadynamics Driver: +https://github.com/MolSSI-MDI/MDI_Metadynamics diff --git a/examples/USER/mdi/Script.sh b/examples/USER/mdi/Script.sh new file mode 100644 index 0000000000..9e3f6e5f62 --- /dev/null +++ b/examples/USER/mdi/Script.sh @@ -0,0 +1,16 @@ +#!/bin/bash +# sample launch scripts + + +# TCP, running LAMMPS on one proc + +python driver.py -mdi "-name driver -role DRIVER -method TCP -port 8021" & +../../../src/lmp_mdi -mdi "-name LAMMPS -role ENGINE -method TCP -port 8021 -hostname localhost" -in lammps.in > lammps.out & +wait + + +# TCP, running LAMMPS on two procs + +python driver.py -mdi "-name driver -role DRIVER -method TCP -port 8021" & +mpiexec -n 2 ../../../src/lmp_mdi -mdi "-name LAMMPS -role ENGINE -method TCP -port 8021 -hostname localhost" -in lammps.in > lammps.out & +wait diff --git a/examples/USER/mdi/driver.py b/examples/USER/mdi/driver.py new file mode 100644 index 0000000000..934f6f7be1 --- /dev/null +++ b/examples/USER/mdi/driver.py @@ -0,0 +1,24 @@ +import sys +import mdi + +use_mpi4py = False +try: + from mpi4py import MPI + use_mpi4py = True +except: + pass + +# Initialize the MDI Library +mdi.MDI_Init(sys.argv[2]) + +# Connect to the engine +comm = mdi.MDI_Accept_communicator() + +# Determine the name of the engine +mdi.MDI_Send_Command(" @@ -46,19 +46,20 @@ AngleZero2::~AngleZero2() void AngleZero2::compute(int eflag, int vflag) { - ev_init(eflag,vflag); + ev_init(eflag, vflag); } /* ---------------------------------------------------------------------- */ void AngleZero2::settings(int narg, char **arg) { - if ((narg != 0) && (narg != 1)) - error->all(FLERR,"Illegal angle_style command"); + if ((narg != 0) && (narg != 1)) error->all(FLERR, "Illegal angle_style command"); if (narg == 1) { - if (strcmp("nocoeff",arg[0]) == 0) coeffflag=0; - else error->all(FLERR,"Illegal angle_style command"); + if (strcmp("nocoeff", arg[0]) == 0) + coeffflag = 0; + else + error->all(FLERR, "Illegal angle_style command"); } } @@ -69,8 +70,8 @@ void AngleZero2::allocate() allocated = 1; int n = atom->nangletypes; - memory->create(theta0,n+1,"angle:theta0"); - memory->create(setflag,n+1,"angle:setflag"); + memory->create(theta0, n + 1, "angle:theta0"); + memory->create(setflag, n + 1, "angle:setflag"); for (int i = 1; i <= n; i++) setflag[i] = 0; } @@ -81,27 +82,26 @@ void AngleZero2::allocate() void AngleZero2::coeff(int narg, char **arg) { if ((narg < 1) || (coeffflag && narg > 2)) - error->all(FLERR,"Incorrect args for angle coefficients"); + error->all(FLERR, "Incorrect args for angle coefficients"); if (!allocated) allocate(); - int ilo,ihi; - utils::bounds(FLERR,arg[0],1,atom->nangletypes,ilo,ihi,error); + int ilo, ihi; + utils::bounds(FLERR, arg[0], 1, atom->nangletypes, ilo, ihi, error); double theta0_one = 0.0; - if (coeffflag && (narg == 2)) - theta0_one = utils::numeric(FLERR,arg[1],false,lmp); + if (coeffflag && (narg == 2)) theta0_one = utils::numeric(FLERR, arg[1], false, lmp); // convert theta0 from degrees to radians int count = 0; for (int i = ilo; i <= ihi; i++) { setflag[i] = 1; - theta0[i] = theta0_one/180.0 * MY_PI; + theta0[i] = theta0_one / 180.0 * MY_PI; count++; } - if (count == 0) error->all(FLERR,"Incorrect args for angle coefficients"); + if (count == 0) error->all(FLERR, "Incorrect args for angle coefficients"); } /* ---------------------------------------------------------------------- */ @@ -115,8 +115,9 @@ double AngleZero2::equilibrium_angle(int i) proc 0 writes out coeffs to restart file ------------------------------------------------------------------------- */ -void AngleZero2::write_restart(FILE *fp) { - fwrite(&theta0[1],sizeof(double),atom->nangletypes,fp); +void AngleZero2::write_restart(FILE *fp) +{ + fwrite(&theta0[1], sizeof(double), atom->nangletypes, fp); } /* ---------------------------------------------------------------------- @@ -128,9 +129,9 @@ void AngleZero2::read_restart(FILE *fp) allocate(); if (comm->me == 0) { - utils::sfread(FLERR,&theta0[1],sizeof(double),atom->nangletypes,fp,nullptr,error); + utils::sfread(FLERR, &theta0[1], sizeof(double), atom->nangletypes, fp, nullptr, error); } - MPI_Bcast(&theta0[1],atom->nangletypes,MPI_DOUBLE,0,world); + MPI_Bcast(&theta0[1], atom->nangletypes, MPI_DOUBLE, 0, world); for (int i = 1; i <= atom->nangletypes; i++) setflag[i] = 1; } @@ -140,8 +141,7 @@ void AngleZero2::read_restart(FILE *fp) void AngleZero2::write_data(FILE *fp) { - for (int i = 1; i <= atom->nangletypes; i++) - fprintf(fp,"%d %g\n",i,theta0[i]/MY_PI*180.0); + for (int i = 1; i <= atom->nangletypes; i++) fprintf(fp, "%d %g\n", i, theta0[i] / MY_PI * 180.0); } /* ---------------------------------------------------------------------- */ diff --git a/examples/plugins/angle_zero2.h b/examples/plugins/angle_zero2.h index 15a16b197f..e4d0c3f9e1 100644 --- a/examples/plugins/angle_zero2.h +++ b/examples/plugins/angle_zero2.h @@ -40,7 +40,7 @@ class AngleZero2 : public Angle { void allocate(); }; -} +} // namespace LAMMPS_NS #endif diff --git a/examples/plugins/bond_zero2.cpp b/examples/plugins/bond_zero2.cpp index b015a60ed3..5dcacea6d7 100644 --- a/examples/plugins/bond_zero2.cpp +++ b/examples/plugins/bond_zero2.cpp @@ -44,19 +44,20 @@ BondZero2::~BondZero2() void BondZero2::compute(int eflag, int vflag) { - ev_init(eflag,vflag); + ev_init(eflag, vflag); } /* ---------------------------------------------------------------------- */ void BondZero2::settings(int narg, char **arg) { - if ((narg != 0) && (narg != 1)) - error->all(FLERR,"Illegal bond_style command"); + if ((narg != 0) && (narg != 1)) error->all(FLERR, "Illegal bond_style command"); if (narg == 1) { - if (strcmp("nocoeff",arg[0]) == 0) coeffflag=0; - else error->all(FLERR,"Illegal bond_style command"); + if (strcmp("nocoeff", arg[0]) == 0) + coeffflag = 0; + else + error->all(FLERR, "Illegal bond_style command"); } } @@ -67,8 +68,8 @@ void BondZero2::allocate() allocated = 1; int n = atom->nbondtypes; - memory->create(r0,n+1,"bond:r0"); - memory->create(setflag,n+1,"bond:setflag"); + memory->create(r0, n + 1, "bond:r0"); + memory->create(setflag, n + 1, "bond:setflag"); for (int i = 1; i <= n; i++) setflag[i] = 0; } @@ -79,16 +80,15 @@ void BondZero2::allocate() void BondZero2::coeff(int narg, char **arg) { if ((narg < 1) || (coeffflag && narg > 2)) - error->all(FLERR,"Incorrect args for bond coefficients"); + error->all(FLERR, "Incorrect args for bond coefficients"); if (!allocated) allocate(); - int ilo,ihi; - utils::bounds(FLERR,arg[0],1,atom->nbondtypes,ilo,ihi,error); + int ilo, ihi; + utils::bounds(FLERR, arg[0], 1, atom->nbondtypes, ilo, ihi, error); double r0_one = 0.0; - if (coeffflag && (narg == 2)) - r0_one = utils::numeric(FLERR,arg[1],false,lmp); + if (coeffflag && (narg == 2)) r0_one = utils::numeric(FLERR, arg[1], false, lmp); int count = 0; for (int i = ilo; i <= ihi; i++) { @@ -97,7 +97,7 @@ void BondZero2::coeff(int narg, char **arg) count++; } - if (count == 0) error->all(FLERR,"Incorrect args for bond coefficients"); + if (count == 0) error->all(FLERR, "Incorrect args for bond coefficients"); } /* ---------------------------------------------------------------------- @@ -113,8 +113,9 @@ double BondZero2::equilibrium_distance(int i) proc 0 writes out coeffs to restart file ------------------------------------------------------------------------- */ -void BondZero2::write_restart(FILE *fp) { - fwrite(&r0[1],sizeof(double),atom->nbondtypes,fp); +void BondZero2::write_restart(FILE *fp) +{ + fwrite(&r0[1], sizeof(double), atom->nbondtypes, fp); } /* ---------------------------------------------------------------------- @@ -126,9 +127,9 @@ void BondZero2::read_restart(FILE *fp) allocate(); if (comm->me == 0) { - utils::sfread(FLERR,&r0[1],sizeof(double),atom->nbondtypes,fp,nullptr,error); + utils::sfread(FLERR, &r0[1], sizeof(double), atom->nbondtypes, fp, nullptr, error); } - MPI_Bcast(&r0[1],atom->nbondtypes,MPI_DOUBLE,0,world); + MPI_Bcast(&r0[1], atom->nbondtypes, MPI_DOUBLE, 0, world); for (int i = 1; i <= atom->nbondtypes; i++) setflag[i] = 1; } @@ -139,14 +140,12 @@ void BondZero2::read_restart(FILE *fp) void BondZero2::write_data(FILE *fp) { - for (int i = 1; i <= atom->nbondtypes; i++) - fprintf(fp,"%d %g\n",i,r0[i]); + for (int i = 1; i <= atom->nbondtypes; i++) fprintf(fp, "%d %g\n", i, r0[i]); } /* ---------------------------------------------------------------------- */ -double BondZero2::single(int /*type*/, double /*rsq*/, int /*i*/, int /*j*/, - double & /*fforce*/) +double BondZero2::single(int /*type*/, double /*rsq*/, int /*i*/, int /*j*/, double & /*fforce*/) { return 0.0; } @@ -156,6 +155,6 @@ double BondZero2::single(int /*type*/, double /*rsq*/, int /*i*/, int /*j*/, void *BondZero2::extract(const char *str, int &dim) { dim = 1; - if (strcmp(str,"r0")==0) return (void*) r0; + if (strcmp(str, "r0") == 0) return (void *) r0; return nullptr; } diff --git a/examples/plugins/bond_zero2.h b/examples/plugins/bond_zero2.h index 37609561a7..a3649ff9f1 100644 --- a/examples/plugins/bond_zero2.h +++ b/examples/plugins/bond_zero2.h @@ -41,7 +41,7 @@ class BondZero2 : public Bond { virtual void allocate(); }; -} +} // namespace LAMMPS_NS #endif diff --git a/examples/plugins/dihedral_zero2.cpp b/examples/plugins/dihedral_zero2.cpp index 00d9817c96..cf07de820c 100644 --- a/examples/plugins/dihedral_zero2.cpp +++ b/examples/plugins/dihedral_zero2.cpp @@ -36,28 +36,27 @@ DihedralZero2::DihedralZero2(LAMMPS *lmp) : Dihedral(lmp), coeffflag(1) DihedralZero2::~DihedralZero2() { - if (allocated && !copymode) { - memory->destroy(setflag); - } + if (allocated && !copymode) { memory->destroy(setflag); } } /* ---------------------------------------------------------------------- */ void DihedralZero2::compute(int eflag, int vflag) { - ev_init(eflag,vflag); + ev_init(eflag, vflag); } /* ---------------------------------------------------------------------- */ void DihedralZero2::settings(int narg, char **arg) { - if ((narg != 0) && (narg != 1)) - error->all(FLERR,"Illegal dihedral_style command"); + if ((narg != 0) && (narg != 1)) error->all(FLERR, "Illegal dihedral_style command"); if (narg == 1) { - if (strcmp("nocoeff",arg[0]) == 0) coeffflag=0; - else error->all(FLERR,"Illegal dihedral_style command"); + if (strcmp("nocoeff", arg[0]) == 0) + coeffflag = 0; + else + error->all(FLERR, "Illegal dihedral_style command"); } } @@ -68,7 +67,7 @@ void DihedralZero2::allocate() allocated = 1; int n = atom->ndihedraltypes; - memory->create(setflag,n+1,"dihedral:setflag"); + memory->create(setflag, n + 1, "dihedral:setflag"); for (int i = 1; i <= n; i++) setflag[i] = 0; } @@ -79,12 +78,12 @@ void DihedralZero2::allocate() void DihedralZero2::coeff(int narg, char **arg) { if ((narg < 1) || (coeffflag && narg > 1)) - error->all(FLERR,"Incorrect args for dihedral coefficients"); + error->all(FLERR, "Incorrect args for dihedral coefficients"); if (!allocated) allocate(); - int ilo,ihi; - utils::bounds(FLERR,arg[0],1,atom->ndihedraltypes,ilo,ihi,error); + int ilo, ihi; + utils::bounds(FLERR, arg[0], 1, atom->ndihedraltypes, ilo, ihi, error); int count = 0; for (int i = ilo; i <= ihi; i++) { @@ -92,7 +91,7 @@ void DihedralZero2::coeff(int narg, char **arg) count++; } - if (count == 0) error->all(FLERR,"Incorrect args for dihedral coefficients"); + if (count == 0) error->all(FLERR, "Incorrect args for dihedral coefficients"); } /* ---------------------------------------------------------------------- @@ -115,7 +114,7 @@ void DihedralZero2::read_restart(FILE * /*fp*/) proc 0 writes to data file ------------------------------------------------------------------------- */ -void DihedralZero2::write_data(FILE *fp) { - for (int i = 1; i <= atom->ndihedraltypes; i++) - fprintf(fp,"%d\n",i); +void DihedralZero2::write_data(FILE *fp) +{ + for (int i = 1; i <= atom->ndihedraltypes; i++) fprintf(fp, "%d\n", i); } diff --git a/examples/plugins/dihedral_zero2.h b/examples/plugins/dihedral_zero2.h index 510e384340..dcd94bd30f 100644 --- a/examples/plugins/dihedral_zero2.h +++ b/examples/plugins/dihedral_zero2.h @@ -40,7 +40,7 @@ class DihedralZero2 : public Dihedral { virtual void allocate(); }; -} +} // namespace LAMMPS_NS #endif diff --git a/examples/plugins/fix_nve2.cpp b/examples/plugins/fix_nve2.cpp index bb6f53b810..bf30219bf2 100644 --- a/examples/plugins/fix_nve2.cpp +++ b/examples/plugins/fix_nve2.cpp @@ -12,23 +12,21 @@ ------------------------------------------------------------------------- */ #include "fix_nve2.h" -#include #include "atom.h" -#include "force.h" -#include "update.h" -#include "respa.h" #include "error.h" +#include "force.h" +#include "respa.h" +#include "update.h" +#include using namespace LAMMPS_NS; using namespace FixConst; /* ---------------------------------------------------------------------- */ -FixNVE2::FixNVE2(LAMMPS *lmp, int narg, char **arg) : - Fix(lmp, narg, arg) +FixNVE2::FixNVE2(LAMMPS *lmp, int narg, char **arg) : Fix(lmp, narg, arg) { - if (strcmp(style,"nve/sphere") != 0 && narg < 3) - error->all(FLERR,"Illegal fix nve command"); + if (strcmp(style, "nve/sphere") != 0 && narg < 3) error->all(FLERR, "Illegal fix nve command"); dynamic_group_allow = 1; time_integrate = 1; @@ -53,8 +51,7 @@ void FixNVE2::init() dtv = update->dt; dtf = 0.5 * update->dt * force->ftm2v; - if (strstr(update->integrate_style,"respa")) - step_respa = ((Respa *) update->integrate)->step; + if (strstr(update->integrate_style, "respa")) step_respa = ((Respa *) update->integrate)->step; } /* ---------------------------------------------------------------------- @@ -150,8 +147,10 @@ void FixNVE2::initial_integrate_respa(int vflag, int ilevel, int /*iloop*/) // innermost level - NVE update of v and x // all other levels - NVE update of v - if (ilevel == 0) initial_integrate(vflag); - else final_integrate(); + if (ilevel == 0) + initial_integrate(vflag); + else + final_integrate(); } /* ---------------------------------------------------------------------- */ diff --git a/examples/plugins/fix_nve2.h b/examples/plugins/fix_nve2.h index f1b2244128..a54949cb5f 100644 --- a/examples/plugins/fix_nve2.h +++ b/examples/plugins/fix_nve2.h @@ -13,7 +13,7 @@ #ifdef FIX_CLASS -FixStyle(nve2,FixNVE2) +FixStyle(nve2, FixNVE2) #else @@ -37,17 +37,17 @@ class FixNVE2 : public Fix { virtual void reset_dt(); protected: - double dtv,dtf; + double dtv, dtf; double *step_respa; int mass_require; }; -} +} // namespace LAMMPS_NS #endif #endif -/* ERROR/WARNING messages: + /* ERROR/WARNING messages: E: Illegal ... command diff --git a/examples/plugins/helloplugin.cpp b/examples/plugins/helloplugin.cpp index f4a49c1a5a..9f3406a282 100644 --- a/examples/plugins/helloplugin.cpp +++ b/examples/plugins/helloplugin.cpp @@ -9,25 +9,24 @@ #include namespace LAMMPS_NS { - class Hello : public Command { - public: - Hello(class LAMMPS *lmp) : Command(lmp) {}; - void command(int, char **); - }; -} +class Hello : public Command { + public: + Hello(class LAMMPS *lmp) : Command(lmp){}; + void command(int, char **); +}; +} // namespace LAMMPS_NS using namespace LAMMPS_NS; void Hello::command(int argc, char **argv) { - if (argc != 1) error->all(FLERR,"Illegal hello command"); - if (comm->me == 0) - utils::logmesg(lmp,fmt::format("Hello, {}!\n",argv[0])); + if (argc != 1) error->all(FLERR, "Illegal hello command"); + if (comm->me == 0) utils::logmesg(lmp, fmt::format("Hello, {}!\n", argv[0])); } static Command *hellocreator(LAMMPS *lmp) { - return new Hello(lmp); + return new Hello(lmp); } extern "C" void lammpsplugin_init(void *lmp, void *handle, void *regfunc) @@ -36,11 +35,11 @@ extern "C" void lammpsplugin_init(void *lmp, void *handle, void *regfunc) lammpsplugin_regfunc register_plugin = (lammpsplugin_regfunc) regfunc; plugin.version = LAMMPS_VERSION; - plugin.style = "command"; - plugin.name = "hello"; - plugin.info = "Hello world command v1.1"; - plugin.author = "Axel Kohlmeyer (akohlmey@gmail.com)"; + plugin.style = "command"; + plugin.name = "hello"; + plugin.info = "Hello world command v1.1"; + plugin.author = "Axel Kohlmeyer (akohlmey@gmail.com)"; plugin.creator.v1 = (lammpsplugin_factory1 *) &hellocreator; - plugin.handle = handle; - (*register_plugin)(&plugin,lmp); + plugin.handle = handle; + (*register_plugin)(&plugin, lmp); } diff --git a/examples/plugins/improper_zero2.cpp b/examples/plugins/improper_zero2.cpp index 2370b32a02..30ee8a8d7d 100644 --- a/examples/plugins/improper_zero2.cpp +++ b/examples/plugins/improper_zero2.cpp @@ -36,28 +36,27 @@ ImproperZero2::ImproperZero2(LAMMPS *lmp) : Improper(lmp), coeffflag(1) ImproperZero2::~ImproperZero2() { - if (allocated && !copymode) { - memory->destroy(setflag); - } + if (allocated && !copymode) { memory->destroy(setflag); } } /* ---------------------------------------------------------------------- */ void ImproperZero2::compute(int eflag, int vflag) { - ev_init(eflag,vflag); + ev_init(eflag, vflag); } /* ---------------------------------------------------------------------- */ void ImproperZero2::settings(int narg, char **arg) { - if ((narg != 0) && (narg != 1)) - error->all(FLERR,"Illegal improper_style command"); + if ((narg != 0) && (narg != 1)) error->all(FLERR, "Illegal improper_style command"); if (narg == 1) { - if (strcmp("nocoeff",arg[0]) == 0) coeffflag=0; - else error->all(FLERR,"Illegal improper_style command"); + if (strcmp("nocoeff", arg[0]) == 0) + coeffflag = 0; + else + error->all(FLERR, "Illegal improper_style command"); } } @@ -68,7 +67,7 @@ void ImproperZero2::allocate() allocated = 1; int n = atom->nimpropertypes; - memory->create(setflag,n+1,"improper:setflag"); + memory->create(setflag, n + 1, "improper:setflag"); for (int i = 1; i <= n; i++) setflag[i] = 0; } @@ -79,12 +78,12 @@ void ImproperZero2::allocate() void ImproperZero2::coeff(int narg, char **arg) { if ((narg < 1) || (coeffflag && narg > 1)) - error->all(FLERR,"Incorrect args for improper coefficients"); + error->all(FLERR, "Incorrect args for improper coefficients"); if (!allocated) allocate(); - int ilo,ihi; - utils::bounds(FLERR,arg[0],1,atom->nimpropertypes,ilo,ihi,error); + int ilo, ihi; + utils::bounds(FLERR, arg[0], 1, atom->nimpropertypes, ilo, ihi, error); int count = 0; for (int i = ilo; i <= ihi; i++) { @@ -92,7 +91,7 @@ void ImproperZero2::coeff(int narg, char **arg) count++; } - if (count == 0) error->all(FLERR,"Incorrect args for improper coefficients"); + if (count == 0) error->all(FLERR, "Incorrect args for improper coefficients"); } /* ---------------------------------------------------------------------- @@ -115,8 +114,7 @@ void ImproperZero2::read_restart(FILE * /*fp*/) proc 0 writes to data file ------------------------------------------------------------------------- */ -void ImproperZero2::write_data(FILE *fp) { - for (int i = 1; i <= atom->nimpropertypes; i++) - fprintf(fp,"%d\n",i); +void ImproperZero2::write_data(FILE *fp) +{ + for (int i = 1; i <= atom->nimpropertypes; i++) fprintf(fp, "%d\n", i); } - diff --git a/examples/plugins/improper_zero2.h b/examples/plugins/improper_zero2.h index 64dd94920a..6c61e4fb08 100644 --- a/examples/plugins/improper_zero2.h +++ b/examples/plugins/improper_zero2.h @@ -36,7 +36,7 @@ class ImproperZero2 : public Improper { virtual void allocate(); }; -} +} // namespace LAMMPS_NS #endif diff --git a/examples/plugins/morse2plugin.cpp b/examples/plugins/morse2plugin.cpp index a4151dcc3d..13dc3507f1 100644 --- a/examples/plugins/morse2plugin.cpp +++ b/examples/plugins/morse2plugin.cpp @@ -12,12 +12,12 @@ using namespace LAMMPS_NS; static Pair *morse2creator(LAMMPS *lmp) { - return new PairMorse2(lmp); + return new PairMorse2(lmp); } static Pair *morse2ompcreator(LAMMPS *lmp) { - return new PairMorse2OMP(lmp); + return new PairMorse2OMP(lmp); } extern "C" void lammpsplugin_init(void *lmp, void *handle, void *regfunc) @@ -27,17 +27,17 @@ extern "C" void lammpsplugin_init(void *lmp, void *handle, void *regfunc) // register plain morse2 pair style plugin.version = LAMMPS_VERSION; - plugin.style = "pair"; - plugin.name = "morse2"; - plugin.info = "Morse2 variant pair style v1.0"; - plugin.author = "Axel Kohlmeyer (akohlmey@gmail.com)"; + plugin.style = "pair"; + plugin.name = "morse2"; + plugin.info = "Morse2 variant pair style v1.0"; + plugin.author = "Axel Kohlmeyer (akohlmey@gmail.com)"; plugin.creator.v1 = (lammpsplugin_factory1 *) &morse2creator; - plugin.handle = handle; - (*register_plugin)(&plugin,lmp); + plugin.handle = handle; + (*register_plugin)(&plugin, lmp); // also register morse2/omp pair style. only need to update changed fields - plugin.name = "morse2/omp"; - plugin.info = "Morse2 variant pair style for OpenMP v1.0"; + plugin.name = "morse2/omp"; + plugin.info = "Morse2 variant pair style for OpenMP v1.0"; plugin.creator.v1 = (lammpsplugin_factory1 *) &morse2ompcreator; - (*register_plugin)(&plugin,lmp); + (*register_plugin)(&plugin, lmp); } diff --git a/examples/plugins/nve2plugin.cpp b/examples/plugins/nve2plugin.cpp index d17db1dfc7..3c622f925c 100644 --- a/examples/plugins/nve2plugin.cpp +++ b/examples/plugins/nve2plugin.cpp @@ -11,7 +11,7 @@ using namespace LAMMPS_NS; static Fix *nve2creator(LAMMPS *lmp, int argc, char **argv) { - return new FixNVE2(lmp, argc, argv); + return new FixNVE2(lmp, argc, argv); } extern "C" void lammpsplugin_init(void *lmp, void *handle, void *regfunc) @@ -20,11 +20,11 @@ extern "C" void lammpsplugin_init(void *lmp, void *handle, void *regfunc) lammpsplugin_regfunc register_plugin = (lammpsplugin_regfunc) regfunc; plugin.version = LAMMPS_VERSION; - plugin.style = "fix"; - plugin.name = "nve2"; - plugin.info = "NVE2 variant fix style v1.0"; - plugin.author = "Axel Kohlmeyer (akohlmey@gmail.com)"; + plugin.style = "fix"; + plugin.name = "nve2"; + plugin.info = "NVE2 variant fix style v1.0"; + plugin.author = "Axel Kohlmeyer (akohlmey@gmail.com)"; plugin.creator.v2 = (lammpsplugin_factory2 *) &nve2creator; - plugin.handle = handle; - (*register_plugin)(&plugin,lmp); + plugin.handle = handle; + (*register_plugin)(&plugin, lmp); } diff --git a/examples/plugins/pair_morse2.cpp b/examples/plugins/pair_morse2.cpp index a235ad297c..ed98dfcf74 100644 --- a/examples/plugins/pair_morse2.cpp +++ b/examples/plugins/pair_morse2.cpp @@ -13,14 +13,14 @@ #include "pair_morse2.h" -#include -#include #include "atom.h" #include "comm.h" -#include "force.h" -#include "neigh_list.h" -#include "memory.h" #include "error.h" +#include "force.h" +#include "memory.h" +#include "neigh_list.h" +#include +#include using namespace LAMMPS_NS; @@ -52,13 +52,13 @@ PairMorse2::~PairMorse2() void PairMorse2::compute(int eflag, int vflag) { - int i,j,ii,jj,inum,jnum,itype,jtype; - double xtmp,ytmp,ztmp,delx,dely,delz,evdwl,fpair; - double rsq,r,dr,dexp,factor_lj; - int *ilist,*jlist,*numneigh,**firstneigh; + int i, j, ii, jj, inum, jnum, itype, jtype; + double xtmp, ytmp, ztmp, delx, dely, delz, evdwl, fpair; + double rsq, r, dr, dexp, factor_lj; + int *ilist, *jlist, *numneigh, **firstneigh; evdwl = 0.0; - ev_init(eflag,vflag); + ev_init(eflag, vflag); double **x = atom->x; double **f = atom->f; @@ -91,32 +91,30 @@ void PairMorse2::compute(int eflag, int vflag) delx = xtmp - x[j][0]; dely = ytmp - x[j][1]; delz = ztmp - x[j][2]; - rsq = delx*delx + dely*dely + delz*delz; + rsq = delx * delx + dely * dely + delz * delz; jtype = type[j]; if (rsq < cutsq[itype][jtype]) { r = sqrt(rsq); dr = r - r0[itype][jtype]; dexp = exp(-alpha[itype][jtype] * dr); - fpair = factor_lj * morse1[itype][jtype] * (dexp*dexp - dexp) / r; + fpair = factor_lj * morse1[itype][jtype] * (dexp * dexp - dexp) / r; - f[i][0] += delx*fpair; - f[i][1] += dely*fpair; - f[i][2] += delz*fpair; + f[i][0] += delx * fpair; + f[i][1] += dely * fpair; + f[i][2] += delz * fpair; if (newton_pair || j < nlocal) { - f[j][0] -= delx*fpair; - f[j][1] -= dely*fpair; - f[j][2] -= delz*fpair; + f[j][0] -= delx * fpair; + f[j][1] -= dely * fpair; + f[j][2] -= delz * fpair; } if (eflag) { - evdwl = d0[itype][jtype] * (dexp*dexp - 2.0*dexp) - - offset[itype][jtype]; + evdwl = d0[itype][jtype] * (dexp * dexp - 2.0 * dexp) - offset[itype][jtype]; evdwl *= factor_lj; } - if (evflag) ev_tally(i,j,nlocal,newton_pair, - evdwl,0.0,fpair,delx,dely,delz); + if (evflag) ev_tally(i, j, nlocal, newton_pair, evdwl, 0.0, fpair, delx, dely, delz); } } } @@ -133,19 +131,18 @@ void PairMorse2::allocate() allocated = 1; int n = atom->ntypes; - memory->create(setflag,n+1,n+1,"pair:setflag"); + memory->create(setflag, n + 1, n + 1, "pair:setflag"); for (int i = 1; i <= n; i++) - for (int j = i; j <= n; j++) - setflag[i][j] = 0; + for (int j = i; j <= n; j++) setflag[i][j] = 0; - memory->create(cutsq,n+1,n+1,"pair:cutsq"); + memory->create(cutsq, n + 1, n + 1, "pair:cutsq"); - memory->create(cut,n+1,n+1,"pair:cut"); - memory->create(d0,n+1,n+1,"pair:d0"); - memory->create(alpha,n+1,n+1,"pair:alpha"); - memory->create(r0,n+1,n+1,"pair:r0"); - memory->create(morse1,n+1,n+1,"pair:morse1"); - memory->create(offset,n+1,n+1,"pair:offset"); + memory->create(cut, n + 1, n + 1, "pair:cut"); + memory->create(d0, n + 1, n + 1, "pair:d0"); + memory->create(alpha, n + 1, n + 1, "pair:alpha"); + memory->create(r0, n + 1, n + 1, "pair:r0"); + memory->create(morse1, n + 1, n + 1, "pair:morse1"); + memory->create(offset, n + 1, n + 1, "pair:offset"); } /* ---------------------------------------------------------------------- @@ -154,14 +151,14 @@ void PairMorse2::allocate() void PairMorse2::settings(int narg, char **arg) { - if (narg != 1) error->all(FLERR,"Illegal pair_style command"); + if (narg != 1) error->all(FLERR, "Illegal pair_style command"); - cut_global = utils::numeric(FLERR,arg[0],false,lmp); + cut_global = utils::numeric(FLERR, arg[0], false, lmp); // reset cutoffs that have been explicitly set if (allocated) { - int i,j; + int i, j; for (i = 1; i <= atom->ntypes; i++) for (j = i; j <= atom->ntypes; j++) if (setflag[i][j]) cut[i][j] = cut_global; @@ -174,24 +171,23 @@ void PairMorse2::settings(int narg, char **arg) void PairMorse2::coeff(int narg, char **arg) { - if (narg < 5 || narg > 6) - error->all(FLERR,"Incorrect args for pair coefficients"); + if (narg < 5 || narg > 6) error->all(FLERR, "Incorrect args for pair coefficients"); if (!allocated) allocate(); - int ilo,ihi,jlo,jhi; - utils::bounds(FLERR,arg[0],1,atom->ntypes,ilo,ihi,error); - utils::bounds(FLERR,arg[1],1,atom->ntypes,jlo,jhi,error); + int ilo, ihi, jlo, jhi; + utils::bounds(FLERR, arg[0], 1, atom->ntypes, ilo, ihi, error); + utils::bounds(FLERR, arg[1], 1, atom->ntypes, jlo, jhi, error); - double d0_one = utils::numeric(FLERR,arg[2],false,lmp); - double alpha_one = utils::numeric(FLERR,arg[3],false,lmp); - double r0_one = utils::numeric(FLERR,arg[4],false,lmp); + double d0_one = utils::numeric(FLERR, arg[2], false, lmp); + double alpha_one = utils::numeric(FLERR, arg[3], false, lmp); + double r0_one = utils::numeric(FLERR, arg[4], false, lmp); double cut_one = cut_global; - if (narg == 6) cut_one = utils::numeric(FLERR,arg[5],false,lmp); + if (narg == 6) cut_one = utils::numeric(FLERR, arg[5], false, lmp); int count = 0; for (int i = ilo; i <= ihi; i++) { - for (int j = MAX(jlo,i); j <= jhi; j++) { + for (int j = MAX(jlo, i); j <= jhi; j++) { d0[i][j] = d0_one; alpha[i][j] = alpha_one; r0[i][j] = r0_one; @@ -201,24 +197,24 @@ void PairMorse2::coeff(int narg, char **arg) } } - if (count == 0) error->all(FLERR,"Incorrect args for pair coefficients"); + if (count == 0) error->all(FLERR, "Incorrect args for pair coefficients"); } - /* ---------------------------------------------------------------------- init for one type pair i,j and corresponding j,i ------------------------------------------------------------------------- */ double PairMorse2::init_one(int i, int j) { - if (setflag[i][j] == 0) error->all(FLERR,"All pair coeffs are not set"); + if (setflag[i][j] == 0) error->all(FLERR, "All pair coeffs are not set"); - morse1[i][j] = 2.0*d0[i][j]*alpha[i][j]; + morse1[i][j] = 2.0 * d0[i][j] * alpha[i][j]; if (offset_flag) { double alpha_dr = -alpha[i][j] * (cut[i][j] - r0[i][j]); - offset[i][j] = d0[i][j] * (exp(2.0*alpha_dr) - 2.0*exp(alpha_dr)); - } else offset[i][j] = 0.0; + offset[i][j] = d0[i][j] * (exp(2.0 * alpha_dr) - 2.0 * exp(alpha_dr)); + } else + offset[i][j] = 0.0; d0[j][i] = d0[i][j]; alpha[j][i] = alpha[i][j]; @@ -237,15 +233,15 @@ void PairMorse2::write_restart(FILE *fp) { write_restart_settings(fp); - int i,j; + int i, j; for (i = 1; i <= atom->ntypes; i++) for (j = i; j <= atom->ntypes; j++) { - fwrite(&setflag[i][j],sizeof(int),1,fp); + fwrite(&setflag[i][j], sizeof(int), 1, fp); if (setflag[i][j]) { - fwrite(&d0[i][j],sizeof(double),1,fp); - fwrite(&alpha[i][j],sizeof(double),1,fp); - fwrite(&r0[i][j],sizeof(double),1,fp); - fwrite(&cut[i][j],sizeof(double),1,fp); + fwrite(&d0[i][j], sizeof(double), 1, fp); + fwrite(&alpha[i][j], sizeof(double), 1, fp); + fwrite(&r0[i][j], sizeof(double), 1, fp); + fwrite(&cut[i][j], sizeof(double), 1, fp); } } } @@ -260,23 +256,23 @@ void PairMorse2::read_restart(FILE *fp) allocate(); - int i,j; + int i, j; int me = comm->me; for (i = 1; i <= atom->ntypes; i++) for (j = i; j <= atom->ntypes; j++) { - if (me == 0) utils::sfread(FLERR,&setflag[i][j],sizeof(int),1,fp,nullptr,error); - MPI_Bcast(&setflag[i][j],1,MPI_INT,0,world); + if (me == 0) utils::sfread(FLERR, &setflag[i][j], sizeof(int), 1, fp, nullptr, error); + MPI_Bcast(&setflag[i][j], 1, MPI_INT, 0, world); if (setflag[i][j]) { if (me == 0) { - utils::sfread(FLERR,&d0[i][j],sizeof(double),1,fp,nullptr,error); - utils::sfread(FLERR,&alpha[i][j],sizeof(double),1,fp,nullptr,error); - utils::sfread(FLERR,&r0[i][j],sizeof(double),1,fp,nullptr,error); - utils::sfread(FLERR,&cut[i][j],sizeof(double),1,fp,nullptr,error); + utils::sfread(FLERR, &d0[i][j], sizeof(double), 1, fp, nullptr, error); + utils::sfread(FLERR, &alpha[i][j], sizeof(double), 1, fp, nullptr, error); + utils::sfread(FLERR, &r0[i][j], sizeof(double), 1, fp, nullptr, error); + utils::sfread(FLERR, &cut[i][j], sizeof(double), 1, fp, nullptr, error); } - MPI_Bcast(&d0[i][j],1,MPI_DOUBLE,0,world); - MPI_Bcast(&alpha[i][j],1,MPI_DOUBLE,0,world); - MPI_Bcast(&r0[i][j],1,MPI_DOUBLE,0,world); - MPI_Bcast(&cut[i][j],1,MPI_DOUBLE,0,world); + MPI_Bcast(&d0[i][j], 1, MPI_DOUBLE, 0, world); + MPI_Bcast(&alpha[i][j], 1, MPI_DOUBLE, 0, world); + MPI_Bcast(&r0[i][j], 1, MPI_DOUBLE, 0, world); + MPI_Bcast(&cut[i][j], 1, MPI_DOUBLE, 0, world); } } } @@ -287,9 +283,9 @@ void PairMorse2::read_restart(FILE *fp) void PairMorse2::write_restart_settings(FILE *fp) { - fwrite(&cut_global,sizeof(double),1,fp); - fwrite(&offset_flag,sizeof(int),1,fp); - fwrite(&mix_flag,sizeof(int),1,fp); + fwrite(&cut_global, sizeof(double), 1, fp); + fwrite(&offset_flag, sizeof(int), 1, fp); + fwrite(&mix_flag, sizeof(int), 1, fp); } /* ---------------------------------------------------------------------- @@ -299,13 +295,13 @@ void PairMorse2::write_restart_settings(FILE *fp) void PairMorse2::read_restart_settings(FILE *fp) { if (comm->me == 0) { - utils::sfread(FLERR,&cut_global,sizeof(double),1,fp,nullptr,error); - utils::sfread(FLERR,&offset_flag,sizeof(int),1,fp,nullptr,error); - utils::sfread(FLERR,&mix_flag,sizeof(int),1,fp,nullptr,error); + utils::sfread(FLERR, &cut_global, sizeof(double), 1, fp, nullptr, error); + utils::sfread(FLERR, &offset_flag, sizeof(int), 1, fp, nullptr, error); + utils::sfread(FLERR, &mix_flag, sizeof(int), 1, fp, nullptr, error); } - MPI_Bcast(&cut_global,1,MPI_DOUBLE,0,world); - MPI_Bcast(&offset_flag,1,MPI_INT,0,world); - MPI_Bcast(&mix_flag,1,MPI_INT,0,world); + MPI_Bcast(&cut_global, 1, MPI_DOUBLE, 0, world); + MPI_Bcast(&offset_flag, 1, MPI_INT, 0, world); + MPI_Bcast(&mix_flag, 1, MPI_INT, 0, world); } /* ---------------------------------------------------------------------- @@ -315,7 +311,7 @@ void PairMorse2::read_restart_settings(FILE *fp) void PairMorse2::write_data(FILE *fp) { for (int i = 1; i <= atom->ntypes; i++) - fprintf(fp,"%d %g %g %g\n",i,d0[i][i],alpha[i][i],r0[i][i]); + fprintf(fp, "%d %g %g %g\n", i, d0[i][i], alpha[i][i], r0[i][i]); } /* ---------------------------------------------------------------------- @@ -326,25 +322,23 @@ void PairMorse2::write_data_all(FILE *fp) { for (int i = 1; i <= atom->ntypes; i++) for (int j = i; j <= atom->ntypes; j++) - fprintf(fp,"%d %d %g %g %g %g\n", - i,j,d0[i][j],alpha[i][j],r0[i][j],cut[i][j]); + fprintf(fp, "%d %d %g %g %g %g\n", i, j, d0[i][j], alpha[i][j], r0[i][j], cut[i][j]); } /* ---------------------------------------------------------------------- */ double PairMorse2::single(int /*i*/, int /*j*/, int itype, int jtype, double rsq, - double /*factor_coul*/, double factor_lj, - double &fforce) + double /*factor_coul*/, double factor_lj, double &fforce) { - double r,dr,dexp,phi; + double r, dr, dexp, phi; r = sqrt(rsq); dr = r - r0[itype][jtype]; dexp = exp(-alpha[itype][jtype] * dr); - fforce = factor_lj * morse1[itype][jtype] * (dexp*dexp - dexp) / r; + fforce = factor_lj * morse1[itype][jtype] * (dexp * dexp - dexp) / r; - phi = d0[itype][jtype] * (dexp*dexp - 2.0*dexp) - offset[itype][jtype]; - return factor_lj*phi; + phi = d0[itype][jtype] * (dexp * dexp - 2.0 * dexp) - offset[itype][jtype]; + return factor_lj * phi; } /* ---------------------------------------------------------------------- */ @@ -352,8 +346,8 @@ double PairMorse2::single(int /*i*/, int /*j*/, int itype, int jtype, double rsq void *PairMorse2::extract(const char *str, int &dim) { dim = 2; - if (strcmp(str,"d0") == 0) return (void *) d0; - if (strcmp(str,"r0") == 0) return (void *) r0; - if (strcmp(str,"alpha") == 0) return (void *) alpha; + if (strcmp(str, "d0") == 0) return (void *) d0; + if (strcmp(str, "r0") == 0) return (void *) r0; + if (strcmp(str, "alpha") == 0) return (void *) alpha; return nullptr; } diff --git a/examples/plugins/pair_morse2.h b/examples/plugins/pair_morse2.h index c20f381c63..472709d574 100644 --- a/examples/plugins/pair_morse2.h +++ b/examples/plugins/pair_morse2.h @@ -39,14 +39,14 @@ class PairMorse2 : public Pair { protected: double cut_global; double **cut; - double **d0,**alpha,**r0; + double **d0, **alpha, **r0; double **morse1; double **offset; virtual void allocate(); }; -} +} // namespace LAMMPS_NS #endif diff --git a/examples/plugins/pair_morse2_omp.cpp b/examples/plugins/pair_morse2_omp.cpp index 14438c764b..11196c001b 100644 --- a/examples/plugins/pair_morse2_omp.cpp +++ b/examples/plugins/pair_morse2_omp.cpp @@ -27,8 +27,7 @@ using namespace LAMMPS_NS; /* ---------------------------------------------------------------------- */ -PairMorse2OMP::PairMorse2OMP(LAMMPS *lmp) : - PairMorse2(lmp), ThrOMP(lmp, THR_PAIR) +PairMorse2OMP::PairMorse2OMP(LAMMPS *lmp) : PairMorse2(lmp), ThrOMP(lmp, THR_PAIR) { suffix_flag |= Suffix::OMP; respa_enable = 0; @@ -38,14 +37,14 @@ PairMorse2OMP::PairMorse2OMP(LAMMPS *lmp) : void PairMorse2OMP::compute(int eflag, int vflag) { - ev_init(eflag,vflag); + ev_init(eflag, vflag); const int nall = atom->nlocal + atom->nghost; const int nthreads = comm->nthreads; const int inum = list->inum; #if defined(_OPENMP) -#pragma omp parallel LMP_DEFAULT_NONE LMP_SHARED(eflag,vflag) +#pragma omp parallel LMP_DEFAULT_NONE LMP_SHARED(eflag, vflag) #endif { int ifrom, ito, tid; @@ -57,38 +56,44 @@ void PairMorse2OMP::compute(int eflag, int vflag) if (evflag) { if (eflag) { - if (force->newton_pair) eval<1,1,1>(ifrom, ito, thr); - else eval<1,1,0>(ifrom, ito, thr); + if (force->newton_pair) + eval<1, 1, 1>(ifrom, ito, thr); + else + eval<1, 1, 0>(ifrom, ito, thr); } else { - if (force->newton_pair) eval<1,0,1>(ifrom, ito, thr); - else eval<1,0,0>(ifrom, ito, thr); + if (force->newton_pair) + eval<1, 0, 1>(ifrom, ito, thr); + else + eval<1, 0, 0>(ifrom, ito, thr); } } else { - if (force->newton_pair) eval<0,0,1>(ifrom, ito, thr); - else eval<0,0,0>(ifrom, ito, thr); + if (force->newton_pair) + eval<0, 0, 1>(ifrom, ito, thr); + else + eval<0, 0, 0>(ifrom, ito, thr); } thr->timer(Timer::PAIR); reduce_thr(this, eflag, vflag, thr); - } // end of omp parallel region + } // end of omp parallel region } template -void PairMorse2OMP::eval(int iifrom, int iito, ThrData * const thr) +void PairMorse2OMP::eval(int iifrom, int iito, ThrData *const thr) { - int i,j,ii,jj,jnum,itype,jtype; - double xtmp,ytmp,ztmp,delx,dely,delz,evdwl,fpair; - double rsq,r,dr,dexp,factor_lj; - int *ilist,*jlist,*numneigh,**firstneigh; + int i, j, ii, jj, jnum, itype, jtype; + double xtmp, ytmp, ztmp, delx, dely, delz, evdwl, fpair; + double rsq, r, dr, dexp, factor_lj; + int *ilist, *jlist, *numneigh, **firstneigh; evdwl = 0.0; - const dbl3_t * _noalias const x = (dbl3_t *) atom->x[0]; - dbl3_t * _noalias const f = (dbl3_t *) thr->get_f()[0]; - const int * _noalias const type = atom->type; + const dbl3_t *_noalias const x = (dbl3_t *) atom->x[0]; + dbl3_t *_noalias const f = (dbl3_t *) thr->get_f()[0]; + const int *_noalias const type = atom->type; const int nlocal = atom->nlocal; - const double * _noalias const special_lj = force->special_lj; - double fxtmp,fytmp,fztmp; + const double *_noalias const special_lj = force->special_lj; + double fxtmp, fytmp, fztmp; ilist = list->ilist; numneigh = list->numneigh; @@ -105,7 +110,7 @@ void PairMorse2OMP::eval(int iifrom, int iito, ThrData * const thr) itype = type[i]; jlist = firstneigh[i]; jnum = numneigh[i]; - fxtmp=fytmp=fztmp=0.0; + fxtmp = fytmp = fztmp = 0.0; for (jj = 0; jj < jnum; jj++) { j = jlist[jj]; @@ -115,32 +120,31 @@ void PairMorse2OMP::eval(int iifrom, int iito, ThrData * const thr) delx = xtmp - x[j].x; dely = ytmp - x[j].y; delz = ztmp - x[j].z; - rsq = delx*delx + dely*dely + delz*delz; + rsq = delx * delx + dely * dely + delz * delz; jtype = type[j]; if (rsq < cutsq[itype][jtype]) { r = sqrt(rsq); dr = r - r0[itype][jtype]; dexp = exp(-alpha[itype][jtype] * dr); - fpair = factor_lj * morse1[itype][jtype] * (dexp*dexp - dexp) / r; + fpair = factor_lj * morse1[itype][jtype] * (dexp * dexp - dexp) / r; - fxtmp += delx*fpair; - fytmp += dely*fpair; - fztmp += delz*fpair; + fxtmp += delx * fpair; + fytmp += dely * fpair; + fztmp += delz * fpair; if (NEWTON_PAIR || j < nlocal) { - f[j].x -= delx*fpair; - f[j].y -= dely*fpair; - f[j].z -= delz*fpair; + f[j].x -= delx * fpair; + f[j].y -= dely * fpair; + f[j].z -= delz * fpair; } if (EFLAG) { - evdwl = d0[itype][jtype] * (dexp*dexp - 2.0*dexp) - - offset[itype][jtype]; + evdwl = d0[itype][jtype] * (dexp * dexp - 2.0 * dexp) - offset[itype][jtype]; evdwl *= factor_lj; } - if (EVFLAG) ev_tally_thr(this,i,j,nlocal,NEWTON_PAIR, - evdwl,0.0,fpair,delx,dely,delz,thr); + if (EVFLAG) + ev_tally_thr(this, i, j, nlocal, NEWTON_PAIR, evdwl, 0.0, fpair, delx, dely, delz, thr); } } f[i].x += fxtmp; diff --git a/examples/plugins/pair_morse2_omp.h b/examples/plugins/pair_morse2_omp.h index c5ed7b5765..47ffd2bda6 100644 --- a/examples/plugins/pair_morse2_omp.h +++ b/examples/plugins/pair_morse2_omp.h @@ -33,9 +33,9 @@ class PairMorse2OMP : public PairMorse2, public ThrOMP { private: template - void eval(int ifrom, int ito, ThrData * const thr); + void eval(int ifrom, int ito, ThrData *const thr); }; -} +} // namespace LAMMPS_NS #endif diff --git a/examples/plugins/pair_zero2.cpp b/examples/plugins/pair_zero2.cpp index d8e23c902d..2979f62da0 100644 --- a/examples/plugins/pair_zero2.cpp +++ b/examples/plugins/pair_zero2.cpp @@ -19,8 +19,8 @@ #include "atom.h" #include "comm.h" -#include "memory.h" #include "error.h" +#include "memory.h" #include @@ -28,11 +28,12 @@ using namespace LAMMPS_NS; /* ---------------------------------------------------------------------- */ -PairZero2::PairZero2(LAMMPS *lmp) : Pair(lmp) { - coeffflag=1; - writedata=1; - single_enable=1; - respa_enable=1; +PairZero2::PairZero2(LAMMPS *lmp) : Pair(lmp) +{ + coeffflag = 1; + writedata = 1; + single_enable = 1; + respa_enable = 1; } /* ---------------------------------------------------------------------- */ @@ -50,15 +51,15 @@ PairZero2::~PairZero2() void PairZero2::compute(int eflag, int vflag) { - ev_init(eflag,vflag); - if (vflag_fdotr) virial_fdotr_compute(); + ev_init(eflag, vflag); + if (vflag_fdotr) virial_fdotr_compute(); } /* ---------------------------------------------------------------------- */ void PairZero2::compute_outer(int eflag, int vflag) { - ev_init(eflag,vflag); + ev_init(eflag, vflag); } /* ---------------------------------------------------------------------- @@ -70,13 +71,12 @@ void PairZero2::allocate() allocated = 1; int n = atom->ntypes; - memory->create(setflag,n+1,n+1,"pair:setflag"); + memory->create(setflag, n + 1, n + 1, "pair:setflag"); for (int i = 1; i <= n; i++) - for (int j = i; j <= n; j++) - setflag[i][j] = 0; + for (int j = i; j <= n; j++) setflag[i][j] = 0; - memory->create(cutsq,n+1,n+1,"pair:cutsq"); - memory->create(cut,n+1,n+1,"pair:cut"); + memory->create(cutsq, n + 1, n + 1, "pair:cutsq"); + memory->create(cut, n + 1, n + 1, "pair:cut"); } /* ---------------------------------------------------------------------- @@ -85,22 +85,22 @@ void PairZero2::allocate() void PairZero2::settings(int narg, char **arg) { - if ((narg != 1) && (narg != 2)) - error->all(FLERR,"Illegal pair_style command"); + if ((narg != 1) && (narg != 2)) error->all(FLERR, "Illegal pair_style command"); - cut_global = utils::numeric(FLERR,arg[0],false,lmp); + cut_global = utils::numeric(FLERR, arg[0], false, lmp); if (narg == 2) { - if (strcmp("nocoeff",arg[1]) == 0) coeffflag=0; - else error->all(FLERR,"Illegal pair_style command"); + if (strcmp("nocoeff", arg[1]) == 0) + coeffflag = 0; + else + error->all(FLERR, "Illegal pair_style command"); } // reset cutoffs that have been explicitly set if (allocated) { - int i,j; + int i, j; for (i = 1; i <= atom->ntypes; i++) - for (j = i+1; j <= atom->ntypes; j++) - cut[i][j] = cut_global; + for (j = i + 1; j <= atom->ntypes; j++) cut[i][j] = cut_global; } } @@ -111,27 +111,27 @@ void PairZero2::settings(int narg, char **arg) void PairZero2::coeff(int narg, char **arg) { if ((narg < 2) || (coeffflag && narg > 3)) - error->all(FLERR,"Incorrect args for pair coefficients"); + error->all(FLERR, "Incorrect args for pair coefficients"); if (!allocated) allocate(); - int ilo,ihi,jlo,jhi; - utils::bounds(FLERR,arg[0],1,atom->ntypes,ilo,ihi,error); - utils::bounds(FLERR,arg[1],1,atom->ntypes,jlo,jhi,error); + int ilo, ihi, jlo, jhi; + utils::bounds(FLERR, arg[0], 1, atom->ntypes, ilo, ihi, error); + utils::bounds(FLERR, arg[1], 1, atom->ntypes, jlo, jhi, error); double cut_one = cut_global; - if (coeffflag && (narg == 3)) cut_one = utils::numeric(FLERR,arg[2],false,lmp); + if (coeffflag && (narg == 3)) cut_one = utils::numeric(FLERR, arg[2], false, lmp); int count = 0; for (int i = ilo; i <= ihi; i++) { - for (int j = MAX(jlo,i); j <= jhi; j++) { + for (int j = MAX(jlo, i); j <= jhi; j++) { cut[i][j] = cut_one; setflag[i][j] = 1; count++; } } - if (count == 0) error->all(FLERR,"Incorrect args for pair coefficients"); + if (count == 0) error->all(FLERR, "Incorrect args for pair coefficients"); } /* ---------------------------------------------------------------------- @@ -140,9 +140,7 @@ void PairZero2::coeff(int narg, char **arg) double PairZero2::init_one(int i, int j) { - if (setflag[i][j] == 0) { - cut[i][j] = mix_distance(cut[i][i],cut[j][j]); - } + if (setflag[i][j] == 0) { cut[i][j] = mix_distance(cut[i][i], cut[j][j]); } return cut[i][j]; } @@ -155,13 +153,11 @@ void PairZero2::write_restart(FILE *fp) { write_restart_settings(fp); - int i,j; + int i, j; for (i = 1; i <= atom->ntypes; i++) for (j = i; j <= atom->ntypes; j++) { - fwrite(&setflag[i][j],sizeof(int),1,fp); - if (setflag[i][j]) { - fwrite(&cut[i][j],sizeof(double),1,fp); - } + fwrite(&setflag[i][j], sizeof(int), 1, fp); + if (setflag[i][j]) { fwrite(&cut[i][j], sizeof(double), 1, fp); } } } @@ -174,17 +170,15 @@ void PairZero2::read_restart(FILE *fp) read_restart_settings(fp); allocate(); - int i,j; + int i, j; int me = comm->me; for (i = 1; i <= atom->ntypes; i++) for (j = i; j <= atom->ntypes; j++) { - if (me == 0) utils::sfread(FLERR,&setflag[i][j],sizeof(int),1,fp,nullptr,error); - MPI_Bcast(&setflag[i][j],1,MPI_INT,0,world); + if (me == 0) utils::sfread(FLERR, &setflag[i][j], sizeof(int), 1, fp, nullptr, error); + MPI_Bcast(&setflag[i][j], 1, MPI_INT, 0, world); if (setflag[i][j]) { - if (me == 0) { - utils::sfread(FLERR,&cut[i][j],sizeof(double),1,fp,nullptr,error); - } - MPI_Bcast(&cut[i][j],1,MPI_DOUBLE,0,world); + if (me == 0) { utils::sfread(FLERR, &cut[i][j], sizeof(double), 1, fp, nullptr, error); } + MPI_Bcast(&cut[i][j], 1, MPI_DOUBLE, 0, world); } } } @@ -195,8 +189,8 @@ void PairZero2::read_restart(FILE *fp) void PairZero2::write_restart_settings(FILE *fp) { - fwrite(&cut_global,sizeof(double),1,fp); - fwrite(&coeffflag,sizeof(int),1,fp); + fwrite(&cut_global, sizeof(double), 1, fp); + fwrite(&coeffflag, sizeof(int), 1, fp); } /* ---------------------------------------------------------------------- @@ -207,11 +201,11 @@ void PairZero2::read_restart_settings(FILE *fp) { int me = comm->me; if (me == 0) { - utils::sfread(FLERR,&cut_global,sizeof(double),1,fp,nullptr,error); - utils::sfread(FLERR,&coeffflag,sizeof(int),1,fp,nullptr,error); + utils::sfread(FLERR, &cut_global, sizeof(double), 1, fp, nullptr, error); + utils::sfread(FLERR, &coeffflag, sizeof(int), 1, fp, nullptr, error); } - MPI_Bcast(&cut_global,1,MPI_DOUBLE,0,world); - MPI_Bcast(&coeffflag,1,MPI_INT,0,world); + MPI_Bcast(&cut_global, 1, MPI_DOUBLE, 0, world); + MPI_Bcast(&coeffflag, 1, MPI_INT, 0, world); } /* ---------------------------------------------------------------------- @@ -220,8 +214,7 @@ void PairZero2::read_restart_settings(FILE *fp) void PairZero2::write_data(FILE *fp) { - for (int i = 1; i <= atom->ntypes; i++) - fprintf(fp,"%d\n",i); + for (int i = 1; i <= atom->ntypes; i++) fprintf(fp, "%d\n", i); } /* ---------------------------------------------------------------------- @@ -231,17 +224,14 @@ void PairZero2::write_data(FILE *fp) void PairZero2::write_data_all(FILE *fp) { for (int i = 1; i <= atom->ntypes; i++) - for (int j = i; j <= atom->ntypes; j++) - fprintf(fp,"%d %d %g\n",i,j,cut[i][j]); + for (int j = i; j <= atom->ntypes; j++) fprintf(fp, "%d %d %g\n", i, j, cut[i][j]); } /* ---------------------------------------------------------------------- */ -double PairZero2::single(int /*i*/, int /*j*/, int /* itype */, int /* jtype */, - double /* rsq */, double /*factor_coul*/, - double /* factor_lj */, double &fforce) +double PairZero2::single(int /*i*/, int /*j*/, int /* itype */, int /* jtype */, double /* rsq */, + double /*factor_coul*/, double /* factor_lj */, double &fforce) { fforce = 0.0; return 0.0; } - diff --git a/examples/plugins/pair_zero2.h b/examples/plugins/pair_zero2.h index 39aa160913..f1b6530dd9 100644 --- a/examples/plugins/pair_zero2.h +++ b/examples/plugins/pair_zero2.h @@ -53,7 +53,7 @@ class PairZero2 : public Pair { virtual void allocate(); }; -} +} // namespace LAMMPS_NS #endif diff --git a/examples/plugins/zero2plugin.cpp b/examples/plugins/zero2plugin.cpp index f181eae781..cd3c024219 100644 --- a/examples/plugins/zero2plugin.cpp +++ b/examples/plugins/zero2plugin.cpp @@ -4,37 +4,37 @@ #include -#include "pair_zero2.h" -#include "bond_zero2.h" #include "angle_zero2.h" +#include "bond_zero2.h" #include "dihedral_zero2.h" #include "improper_zero2.h" +#include "pair_zero2.h" using namespace LAMMPS_NS; static Pair *pairzerocreator(LAMMPS *lmp) { - return new PairZero2(lmp); + return new PairZero2(lmp); } static Bond *bondzerocreator(LAMMPS *lmp) { - return new BondZero2(lmp); + return new BondZero2(lmp); } static Angle *anglezerocreator(LAMMPS *lmp) { - return new AngleZero2(lmp); + return new AngleZero2(lmp); } static Dihedral *dihedralzerocreator(LAMMPS *lmp) { - return new DihedralZero2(lmp); + return new DihedralZero2(lmp); } static Improper *improperzerocreator(LAMMPS *lmp) { - return new ImproperZero2(lmp); + return new ImproperZero2(lmp); } extern "C" void lammpsplugin_init(void *lmp, void *handle, void *regfunc) @@ -44,35 +44,35 @@ extern "C" void lammpsplugin_init(void *lmp, void *handle, void *regfunc) // register zero2 pair style plugin.version = LAMMPS_VERSION; - plugin.style = "pair"; - plugin.name = "zero2"; - plugin.info = "Zero2 variant pair style v1.0"; - plugin.author = "Axel Kohlmeyer (akohlmey@gmail.com)"; + plugin.style = "pair"; + plugin.name = "zero2"; + plugin.info = "Zero2 variant pair style v1.0"; + plugin.author = "Axel Kohlmeyer (akohlmey@gmail.com)"; plugin.creator.v1 = (lammpsplugin_factory1 *) &pairzerocreator; - plugin.handle = handle; - (*register_plugin)(&plugin,lmp); + plugin.handle = handle; + (*register_plugin)(&plugin, lmp); // register zero2 bond style - plugin.style = "bond"; - plugin.info = "Zero2 variant bond style v1.0"; + plugin.style = "bond"; + plugin.info = "Zero2 variant bond style v1.0"; plugin.creator.v1 = (lammpsplugin_factory1 *) &bondzerocreator; - (*register_plugin)(&plugin,lmp); + (*register_plugin)(&plugin, lmp); // register zero2 angle style - plugin.style = "angle"; - plugin.info = "Zero2 variant angle style v1.0"; + plugin.style = "angle"; + plugin.info = "Zero2 variant angle style v1.0"; plugin.creator.v1 = (lammpsplugin_factory1 *) &anglezerocreator; - (*register_plugin)(&plugin,lmp); + (*register_plugin)(&plugin, lmp); // register zero2 dihedral style - plugin.style = "dihedral"; - plugin.info = "Zero2 variant dihedral style v1.0"; + plugin.style = "dihedral"; + plugin.info = "Zero2 variant dihedral style v1.0"; plugin.creator.v1 = (lammpsplugin_factory1 *) &dihedralzerocreator; - (*register_plugin)(&plugin,lmp); + (*register_plugin)(&plugin, lmp); // register zero2 improper style - plugin.style = "improper"; - plugin.info = "Zero2 variant improper style v1.0"; + plugin.style = "improper"; + plugin.info = "Zero2 variant improper style v1.0"; plugin.creator.v1 = (lammpsplugin_factory1 *) &improperzerocreator; - (*register_plugin)(&plugin,lmp); + (*register_plugin)(&plugin, lmp); } diff --git a/examples/python/in.fix_python_invoke_neighlist b/examples/python/in.fix_python_invoke_neighlist index af0399ae1f..f45a92c510 100644 --- a/examples/python/in.fix_python_invoke_neighlist +++ b/examples/python/in.fix_python_invoke_neighlist @@ -33,7 +33,7 @@ def post_force_callback(lmp, v): print(pid_prefix, "### POST_FORCE ###", t) #mylist = L.numpy.get_neighlist(0) - idx = L.find_pair_neighlist("lj/cut", request=0) + idx = L.find_pair_neighlist("lj/cut", reqid=0) mylist = L.numpy.get_neighlist(idx) print(pid_prefix, mylist) nlocal = L.extract_global("nlocal") diff --git a/examples/python/py_nve.py b/examples/python/py_nve.py index 9ff7c0978b..740668b850 100644 --- a/examples/python/py_nve.py +++ b/examples/python/py_nve.py @@ -1,7 +1,5 @@ from __future__ import print_function -from lammps import lammps, LAMMPS_INT, LAMMPS_DOUBLE -import ctypes -import traceback +from lammps import lammps import numpy as np class LAMMPSFix(object): @@ -80,36 +78,30 @@ class NVE_Opt(LAMMPSFixMove): self.ntypes = self.lmp.extract_global("ntypes") self.dtv = dt self.dtf = 0.5 * dt * ftm2v - self.mass = self.lmp.numpy.extract_atom("mass") def initial_integrate(self, vflag): - nlocal = self.lmp.extract_global("nlocal") + mass = self.lmp.numpy.extract_atom("mass") atype = self.lmp.numpy.extract_atom("type") x = self.lmp.numpy.extract_atom("x") v = self.lmp.numpy.extract_atom("v") f = self.lmp.numpy.extract_atom("f") dtf = self.dtf dtv = self.dtv - mass = self.mass dtfm = dtf / np.take(mass, atype) - dtfm.reshape((nlocal, 1)) for d in range(x.shape[1]): - v[:,d] += dtfm[:,0] * f[:,d] + v[:,d] += dtfm * f[:,d] x[:,d] += dtv * v[:,d] def final_integrate(self): - nlocal = self.lmp.extract_global("nlocal") mass = self.lmp.numpy.extract_atom("mass") atype = self.lmp.numpy.extract_atom("type") v = self.lmp.numpy.extract_atom("v") f = self.lmp.numpy.extract_atom("f") dtf = self.dtf - mass = self.mass dtfm = dtf / np.take(mass, atype) - dtfm.reshape((nlocal, 1)) for d in range(v.shape[1]): - v[:,d] += dtfm[:,0] * f[:,d] + v[:,d] += dtfm * f[:,d] diff --git a/examples/srd/in.srd.mixture b/examples/srd/in.srd.mixture index ff6e6f5b72..6a0100d800 100644 --- a/examples/srd/in.srd.mixture +++ b/examples/srd/in.srd.mixture @@ -62,7 +62,7 @@ delete_atoms overlap 0.5 small big reset_timestep 0 -neighbor 0.3 bin +neighbor 0.3 multi neigh_modify delay 0 every 1 check yes comm_modify mode multi group big vel yes diff --git a/lib/atc/LammpsInterface.cpp b/lib/atc/LammpsInterface.cpp index 5727af1904..b123331ee7 100644 --- a/lib/atc/LammpsInterface.cpp +++ b/lib/atc/LammpsInterface.cpp @@ -26,6 +26,7 @@ #include "bond.h" // bond potentials #include "comm.h" // #include "fix.h" +#include "utils.h" // ATC includes #include "ATC_Error.h" @@ -47,6 +48,7 @@ using std::pair; using std::string; using std::set; using LAMMPS_NS::bigint; +using LAMMPS_NS::utils::read_lines_from_file; namespace ATC { @@ -236,7 +238,7 @@ std::string LammpsInterface::read_file(std::string filename) const std::stringstream s; bool eof = false; while ( ! eof) { - eof = lammps_->comm->read_lines_from_file(fp,1,MAXLINE,buffer); + eof = read_lines_from_file(fp,1,MAXLINE,buffer,comm_rank(),lammps_->world); s << buffer; } fclose(fp); diff --git a/lib/gpu/geryon/nvd_device.h b/lib/gpu/geryon/nvd_device.h index 87d5faaf48..52b2ed478e 100644 --- a/lib/gpu/geryon/nvd_device.h +++ b/lib/gpu/geryon/nvd_device.h @@ -24,10 +24,6 @@ #ifndef NVD_DEVICE #define NVD_DEVICE -// workaround after GPU package Feb2021 update -// todo: make new neighbor code work with CUDA -#define LAL_USE_OLD_NEIGHBOR - #include #include #include diff --git a/lib/gpu/geryon/nvd_kernel.h b/lib/gpu/geryon/nvd_kernel.h index c31b8cdf9b..798b12e53c 100644 --- a/lib/gpu/geryon/nvd_kernel.h +++ b/lib/gpu/geryon/nvd_kernel.h @@ -108,17 +108,14 @@ class UCL_Program { std::cerr << log << std::endl << "----------------------------------------------------------\n\n"; #endif - if (foutput != NULL) { - fprintf(foutput,"\n\n"); - fprintf(foutput, - "----------------------------------------------------------\n"); - fprintf(foutput," UCL Error: Error compiling PTX Program...\n"); - fprintf(foutput, - "----------------------------------------------------------\n"); - fprintf(foutput,"%s\n",log); - fprintf(foutput, - "----------------------------------------------------------\n"); - fprintf(foutput,"\n\n"); + if (foutput != nullptr) { + fprintf(foutput,"\n\n"); + fprintf(foutput, "----------------------------------------------------------\n"); + fprintf(foutput, " UCL Error: Error compiling PTX Program...\n"); + fprintf(foutput, "----------------------------------------------------------\n"); + fprintf(foutput, "%s\n",log->c_str()); + fprintf(foutput, "----------------------------------------------------------\n"); + fprintf(foutput,"\n\n"); } return UCL_COMPILE_ERROR; } diff --git a/lib/gpu/geryon/ocl_device.h b/lib/gpu/geryon/ocl_device.h index 4e9e42a3cf..a8e5020f00 100644 --- a/lib/gpu/geryon/ocl_device.h +++ b/lib/gpu/geryon/ocl_device.h @@ -29,7 +29,7 @@ #include #ifndef CL_TARGET_OPENCL_VERSION -#define CL_TARGET_OPENCL_VERSION 210 +#define CL_TARGET_OPENCL_VERSION 300 #endif #ifdef __APPLE__ diff --git a/lib/gpu/geryon/ocl_macros.h b/lib/gpu/geryon/ocl_macros.h index 0e9ce78389..5e5a190ede 100644 --- a/lib/gpu/geryon/ocl_macros.h +++ b/lib/gpu/geryon/ocl_macros.h @@ -5,7 +5,7 @@ #include #ifndef CL_TARGET_OPENCL_VERSION -#define CL_TARGET_OPENCL_VERSION 210 +#define CL_TARGET_OPENCL_VERSION 300 #endif #ifdef __APPLE__ diff --git a/lib/gpu/lal_device.cpp b/lib/gpu/lal_device.cpp index b42aa8e21d..e2b5b9cdb5 100644 --- a/lib/gpu/lal_device.cpp +++ b/lib/gpu/lal_device.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #if (LAL_USE_OMP == 1) #include #endif @@ -777,6 +778,7 @@ void DeviceT::output_times(UCL_Timer &time_pair, Answer &ans, #ifdef USE_OPENCL // Workaround for timing issue on Intel OpenCL if (times[3] > 80e6) times[3]=0.0; + if (times[5] > 80e6) times[5]=0.0; #endif if (replica_me()==0) @@ -1025,6 +1027,22 @@ Device global_device; } using namespace LAMMPS_AL; + +bool lmp_has_gpu_device() +{ + UCL_Device gpu; + return (gpu.num_platforms() > 0); +} + +std::string lmp_gpu_device_info() +{ + std::ostringstream out; + UCL_Device gpu; + if (gpu.num_platforms() > 0) + gpu.print_all(out); + return out.str(); +} + int lmp_init_device(MPI_Comm world, MPI_Comm replica, const int ngpu, const int first_gpu_id, const int gpu_mode, const double particle_split, const int t_per_atom, diff --git a/lib/gpu/lal_eam.cu b/lib/gpu/lal_eam.cu index 3955f3cc8a..0003f76716 100644 --- a/lib/gpu/lal_eam.cu +++ b/lib/gpu/lal_eam.cu @@ -225,7 +225,7 @@ __kernel void k_energy(const __global numtyp4 *restrict x_, const numtyp rdr, const numtyp rdrho, const numtyp rhomax, const int nrho, const int nr, const int t_per_atom) { - int tid, ii, offset, i, itype; + int tid, ii, offset, i, itype, tfrho; atom_info(t_per_atom,ii,tid,offset); int n_stride; @@ -242,6 +242,7 @@ __kernel void k_energy(const __global numtyp4 *restrict x_, numtyp4 ix; fetch4(ix,i,pos_tex); //x_[i]; itype=ix.w; + tfrho=type2frho[itype]; for ( ; nbor +namespace LAMMPS_AL { +#define LJSMOOTHT LJSMOOTH + +extern Device device; + +template +LJSMOOTHT::LJSMOOTH() : BaseAtomic(), _allocated(false) { +} + +template +LJSMOOTHT::~LJSMOOTH() { + clear(); +} + +template +int LJSMOOTHT::bytes_per_atom(const int max_nbors) const { + return this->bytes_per_atom_atomic(max_nbors); +} + +template +int LJSMOOTHT::init(const int ntypes, + double **host_cutsq, double **host_lj1, + double **host_lj2, double **host_lj3, + double **host_lj4, double **host_offset, + double *host_special_lj, const int nlocal, + const int nall, const int max_nbors, + const int maxspecial, const double cell_size, + const double gpu_split, FILE *_screen, + double **host_ljsw0, double **host_ljsw1, double **host_ljsw2, double **host_ljsw3, + double **host_ljsw4, + double **cut_inner, double **cut_inner_sq) { + const int max_shared_types=this->device->max_shared_types(); + + int onetype=0; + #ifdef USE_OPENCL + if (maxspecial==0) + for (int i=1; i0) { + if (onetype>0) + onetype=-1; + else if (onetype==0) + onetype=i*max_shared_types+j; + } + if (onetype<0) onetype=0; + #endif + + int success; + success=this->init_atomic(nlocal,nall,max_nbors,maxspecial,cell_size,gpu_split, + _screen,lj_smooth,"k_lj_smooth",onetype); + if (success!=0) + return success; + + // If atom type constants fit in shared memory use fast kernel + int lj_types=ntypes; + shared_types=false; + if (lj_types<=max_shared_types && this->_block_size>=max_shared_types) { + lj_types=max_shared_types; + shared_types=true; + } + _lj_types=lj_types; + + // Allocate a host write buffer for data initialization + UCL_H_Vec host_write(lj_types*lj_types*32,*(this->ucl_device), + UCL_WRITE_ONLY); + + for (int i=0; iucl_device),UCL_READ_ONLY); + this->atom->type_pack4(ntypes,lj_types,lj1,host_write,host_lj1,host_lj2, + host_cutsq, cut_inner_sq); + + lj3.alloc(lj_types*lj_types,*(this->ucl_device),UCL_READ_ONLY); + this->atom->type_pack4(ntypes,lj_types,lj3,host_write,host_lj3,host_lj4, + host_offset); + + ljsw.alloc(lj_types*lj_types,*(this->ucl_device),UCL_READ_ONLY); + this->atom->type_pack4(ntypes,lj_types,ljsw,host_write,host_ljsw1,host_ljsw2, + host_ljsw3,host_ljsw4); + + ljsw0.alloc(lj_types*lj_types,*(this->ucl_device),UCL_READ_ONLY); + this->atom->type_pack2(ntypes,lj_types,ljsw0,host_write,host_ljsw0,cut_inner); + + UCL_H_Vec dview; + sp_lj.alloc(4,*(this->ucl_device),UCL_READ_ONLY); + dview.view(host_special_lj,4,*(this->ucl_device)); + ucl_copy(sp_lj,dview,false); + + _allocated=true; + this->_max_bytes=lj1.row_bytes()+lj3.row_bytes()+ljsw.row_bytes()+ljsw0.row_bytes()+sp_lj.row_bytes(); + return 0; +} + +template +void LJSMOOTHT::reinit(const int ntypes, double **host_cutsq, double **host_lj1, + double **host_lj2, double **host_lj3, + double **host_lj4, double **host_offset, + double **host_ljsw0, double **host_ljsw1, double **host_ljsw2, double **host_ljsw3, + double **host_ljsw4, + double **cut_inner, double **cut_inner_sq) { + // Allocate a host write buffer for data initialization + UCL_H_Vec host_write(_lj_types*_lj_types*32,*(this->ucl_device), + UCL_WRITE_ONLY); + + for (int i=0; i<_lj_types*_lj_types; i++) + host_write[i]=0.0; + + this->atom->type_pack4(ntypes,_lj_types,lj1,host_write,host_lj1,host_lj2, + host_cutsq,cut_inner_sq); + this->atom->type_pack4(ntypes,_lj_types,lj3,host_write,host_lj3,host_lj4, + host_offset); + this->atom->type_pack4(ntypes,_lj_types,ljsw,host_write,host_ljsw1,host_ljsw2, + host_ljsw3,host_ljsw4); + this->atom->type_pack2(ntypes,_lj_types,ljsw0,host_write,host_ljsw0,cut_inner); +} + +template +void LJSMOOTHT::clear() { + if (!_allocated) + return; + _allocated=false; + + lj1.clear(); + lj3.clear(); + ljsw.clear(); + ljsw0.clear(); + sp_lj.clear(); + this->clear_atomic(); +} + +template +double LJSMOOTHT::host_memory_usage() const { + return this->host_memory_usage_atomic()+sizeof(LJSMOOTH); +} + +// --------------------------------------------------------------------------- +// Calculate energies, forces, and torques +// --------------------------------------------------------------------------- +template +int LJSMOOTHT::loop(const int eflag, const int vflag) { + // Compute the block size and grid size to keep all cores busy + const int BX=this->block_size(); + + int GX=static_cast(ceil(static_cast(this->ans->inum())/ + (BX/this->_threads_per_atom))); + + int ainum=this->ans->inum(); + int nbor_pitch=this->nbor->nbor_pitch(); + this->time_pair.start(); + if (shared_types) { + this->k_pair_sel->set_size(GX,BX); + this->k_pair_sel->run(&this->atom->x, &lj1, &lj3, &ljsw, &ljsw0, &sp_lj, + &this->nbor->dev_nbor, &this->_nbor_data->begin(), + &this->ans->force, &this->ans->engv, &eflag, + &vflag, &ainum, &nbor_pitch, + &this->_threads_per_atom); + } else { + this->k_pair.set_size(GX,BX); + this->k_pair.run(&this->atom->x, &lj1, &lj3, &ljsw, &ljsw0, &_lj_types, &sp_lj, + &this->nbor->dev_nbor, &this->_nbor_data->begin(), + &this->ans->force, &this->ans->engv, &eflag, &vflag, + &ainum, &nbor_pitch, &this->_threads_per_atom); + } + this->time_pair.stop(); + return GX; +} + +template class LJSMOOTH; +} diff --git a/lib/gpu/lal_lj_smooth.cu b/lib/gpu/lal_lj_smooth.cu new file mode 100644 index 0000000000..d4a99ed3a7 --- /dev/null +++ b/lib/gpu/lal_lj_smooth.cu @@ -0,0 +1,259 @@ +// ************************************************************************** +// lj_smooth.cu +// ------------------- +// Gurgen Melikyan (HSE University) +// +// Device code for acceleration of the lj/smooth pair style +// +// __________________________________________________________________________ +// This file is part of the LAMMPS Accelerator Library (LAMMPS_AL) +// __________________________________________________________________________ +// +// begin : +// email : gkmeliyan@edu.hse.ru +// *************************************************************************** + +#if defined(NV_KERNEL) || defined(USE_HIP) +#include "lal_aux_fun1.h" +#ifndef _DOUBLE_DOUBLE +_texture( pos_tex,float4); +#else +_texture_2d( pos_tex,int4); +#endif +#else +#define pos_tex x_ +#endif + +__kernel void k_lj_smooth(const __global numtyp4 *restrict x_, + const __global numtyp4 *restrict lj1, + const __global numtyp4 *restrict lj3, + const __global numtyp4 *restrict ljsw, + const __global numtyp2 *restrict ljsw0, + const int lj_types, + const __global numtyp *restrict sp_lj, + const __global int * dev_nbor, + const __global int * dev_packed, + __global acctyp4 *restrict ans, + __global acctyp *restrict engv, + const int eflag, const int vflag, const int inum, + const int nbor_pitch, const int t_per_atom) { + int tid, ii, offset; + atom_info(t_per_atom,ii,tid,offset); + + int n_stride; + local_allocate_store_pair(); + + acctyp4 f; + f.x=(acctyp)0; f.y=(acctyp)0; f.z=(acctyp)0; + acctyp energy, virial[6]; + if (EVFLAG) { + energy=(acctyp)0; + for (int i=0; i<6; i++) virial[i]=(acctyp)0; + } + + if (ii +class LJSMOOTH : public BaseAtomic { + public: + LJSMOOTH(); + ~LJSMOOTH(); + + /// Clear any previous data and set up for a new LAMMPS run + /** \param max_nbors initial number of rows in the neighbor matrix + * \param cell_size cutoff + skin + * \param gpu_split fraction of particles handled by device + * + * Returns: + * - 0 if successful + * - -1 if fix gpu not found + * - -3 if there is an out of memory error + * - -4 if the GPU library was not compiled for GPU + * - -5 Double precision is not supported on card **/ + int init(const int ntypes, double **host_cutsq, + double **host_lj1, double **host_lj2, double **host_lj3, + double **host_lj4, double **host_offset, double *host_special_lj, + const int nlocal, const int nall, const int max_nbors, + const int maxspecial, const double cell_size, + const double gpu_split, FILE *screen, + double **host_ljsw0, double **host_ljsw1, double **host_ljsw2, + double **host_ljsw3, double **host_ljsw4, + double **cut_inner, double **cut_inner_sq); + + /// Send updated coeffs from host to device (to be compatible with fix adapt) + void reinit(const int ntypes, double **host_cutsq, + double **host_lj1, double **host_lj2, double **host_lj3, + double **host_lj4, double **host_offset, + double **host_ljsw0, double **host_ljsw1, double **host_ljsw2, + double **host_ljsw3, double **host_ljsw4, + double **cut_inner, double **cut_inner_sq); + + /// Clear all host and device data + /** \note This is called at the beginning of the init() routine **/ + void clear(); + + /// Returns memory usage on device per atom + int bytes_per_atom(const int max_nbors) const; + + /// Total host memory used by library for pair style + double host_memory_usage() const; + + // --------------------------- TYPE DATA -------------------------- + + /// lj1.x = lj1, lj1.y = lj2, lj1.z = cutsq, lj1.w = cut_inner_sq + UCL_D_Vec lj1; + /// lj3.x = lj3, lj3.y = lj4, lj3.z = offset + UCL_D_Vec lj3; + /// ljsw.x = ljsw1, ljsw.y = ljsw2, ljsw.z = ljsw3, ljsw.w = ljsw4 + UCL_D_Vec ljsw; + /// ljsw0.x = ljsw0 ljsw0.y = cut_inner + UCL_D_Vec ljsw0; + /// Special LJ values + UCL_D_Vec sp_lj; + + /// If atom type constants fit in shared memory, use fast kernels + bool shared_types; + + /// Number of atom types + int _lj_types; + + private: + bool _allocated; + int loop(const int _eflag, const int _vflag); +}; + +} + +#endif diff --git a/lib/gpu/lal_lj_smooth_ext.cpp b/lib/gpu/lal_lj_smooth_ext.cpp new file mode 100644 index 0000000000..48dad74071 --- /dev/null +++ b/lib/gpu/lal_lj_smooth_ext.cpp @@ -0,0 +1,144 @@ +/*************************************************************************** + lj_smooth_ext.cpp + ------------------- + Gurgen Melikyan (HSE University) + + Functions for LAMMPS access to lj/smooth acceleration routines. + + __________________________________________________________________________ + This file is part of the LAMMPS Accelerator Library (LAMMPS_AL) + __________________________________________________________________________ + + begin : + email : gkmeliyan@edu.hse.ru + ***************************************************************************/ + +#include +#include +#include + +#include "lal_lj_smooth.h" + +using namespace std; +using namespace LAMMPS_AL; + +static LJSMOOTH LJSMTMF; + +// --------------------------------------------------------------------------- +// Allocate memory on host and device and copy constants to device +// --------------------------------------------------------------------------- +int ljsmt_gpu_init(const int ntypes, double **cutsq, double **host_lj1, + double **host_lj2, double **host_lj3, double **host_lj4, + double **offset, double *special_lj, const int inum, + const int nall, const int max_nbors, const int maxspecial, + const double cell_size, int &gpu_mode, FILE *screen, + double **host_ljsw0, double **host_ljsw1, double **host_ljsw2, double **host_ljsw3, + double **host_ljsw4, double **cut_inner, double **cut_inner_sq) { + LJSMTMF.clear(); + gpu_mode=LJSMTMF.device->gpu_mode(); + double gpu_split=LJSMTMF.device->particle_split(); + int first_gpu=LJSMTMF.device->first_device(); + int last_gpu=LJSMTMF.device->last_device(); + int world_me=LJSMTMF.device->world_me(); + int gpu_rank=LJSMTMF.device->gpu_rank(); + int procs_per_gpu=LJSMTMF.device->procs_per_gpu(); + + LJSMTMF.device->init_message(screen,"lj/smooth",first_gpu,last_gpu); + + bool message=false; + if (LJSMTMF.device->replica_me()==0 && screen) + message=true; + + if (message) { + fprintf(screen,"Initializing Device and compiling on process 0..."); + fflush(screen); + } + + int init_ok=0; + if (world_me==0) + init_ok=LJSMTMF.init(ntypes, cutsq, host_lj1, host_lj2, host_lj3, + host_lj4, offset, special_lj, inum, nall, max_nbors, + maxspecial, cell_size, gpu_split, screen, + host_ljsw0, host_ljsw1, host_ljsw2, host_ljsw3, host_ljsw4, cut_inner, cut_inner_sq); + + LJSMTMF.device->world_barrier(); + if (message) + fprintf(screen,"Done.\n"); + + for (int i=0; igpu_barrier(); + if (message) + fprintf(screen,"Done.\n"); + } + if (message) + fprintf(screen,"\n"); + + if (init_ok==0) + LJSMTMF.estimate_gpu_overhead(); + return init_ok; +} + +// --------------------------------------------------------------------------- +// Copy updated coeffs from host to device +// --------------------------------------------------------------------------- +void ljsmt_gpu_reinit(const int ntypes, double **cutsq, double **host_lj1, + double **host_lj2, double **host_lj3, double **host_lj4, + double **offset, double **host_ljsw0, double **host_ljsw1, double **host_ljsw2, double **host_ljsw3, + double **host_ljsw4, double **cut_inner, double **cut_inner_sq) { + int world_me=LJSMTMF.device->world_me(); + int gpu_rank=LJSMTMF.device->gpu_rank(); + int procs_per_gpu=LJSMTMF.device->procs_per_gpu(); + + if (world_me==0) + LJSMTMF.reinit(ntypes, cutsq, host_lj1, host_lj2, host_lj3, host_lj4, offset, host_ljsw0, host_ljsw1, host_ljsw2, host_ljsw3, host_ljsw4, cut_inner, cut_inner_sq); + LJSMTMF.device->world_barrier(); + + for (int i=0; igpu_barrier(); + } +} + +void ljsmt_gpu_clear() { + LJSMTMF.clear(); +} + +int ** ljsmt_gpu_compute_n(const int ago, const int inum_full, + const int nall, double **host_x, int *host_type, + double *sublo, double *subhi, tagint *tag, int **nspecial, + tagint **special, const bool eflag, const bool vflag, + const bool eatom, const bool vatom, int &host_start, + int **ilist, int **jnum, const double cpu_time, + bool &success) { + return LJSMTMF.compute(ago, inum_full, nall, host_x, host_type, sublo, + subhi, tag, nspecial, special, eflag, vflag, eatom, + vatom, host_start, ilist, jnum, cpu_time, success); +} + +void ljsmt_gpu_compute(const int ago, const int inum_full, const int nall, + double **host_x, int *host_type, int *ilist, int *numj, + int **firstneigh, const bool eflag, const bool vflag, + const bool eatom, const bool vatom, int &host_start, + const double cpu_time, bool &success) { + LJSMTMF.compute(ago,inum_full,nall,host_x,host_type,ilist,numj, + firstneigh,eflag,vflag,eatom,vatom,host_start,cpu_time,success); +} + +double ljsmt_gpu_bytes() { + return LJSMTMF.host_memory_usage(); +} diff --git a/lib/gpu/lal_neighbor_gpu.cu b/lib/gpu/lal_neighbor_gpu.cu index b6db97f68a..62c93e6cf1 100644 --- a/lib/gpu/lal_neighbor_gpu.cu +++ b/lib/gpu/lal_neighbor_gpu.cu @@ -115,7 +115,7 @@ __kernel void kernel_calc_cell_counts(const unsigned *restrict cell_id, #define tagint int #endif #ifdef LAMMPS_BIGBIG -#define tagint long long int +#define tagint long #endif #ifdef LAMMPS_SMALLSMALL #define tagint int diff --git a/lib/kokkos/CHANGELOG.md b/lib/kokkos/CHANGELOG.md index c759181aa2..3ce38c37d8 100644 --- a/lib/kokkos/CHANGELOG.md +++ b/lib/kokkos/CHANGELOG.md @@ -1,5 +1,168 @@ # Change Log +## [3.4.00](https://github.com/kokkos/kokkos/tree/3.4.00) (2021-04-25) +[Full Changelog](https://github.com/kokkos/kokkos/compare/3.3.01...3.4.00) + +**Highlights:** +- SYCL Backend Almost Feature Complete +- OpenMPTarget Backend Almost Feature Complete +- Performance Improvements for HIP backend +- Require CMake 3.16 or newer +- Tool Callback Interface Enhancements +- cmath wrapper functions available now in Kokkos::Experimental + +**Features:** +- Implement parallel_scan with ThreadVectorRange and Reducer [\#3861](https://github.com/kokkos/kokkos/pull/3861) +- Implement SYCL Random [\#3849](https://github.com/kokkos/kokkos/pull/3849) +- OpenMPTarget: Adding Implementation for nested reducers [\#3845](https://github.com/kokkos/kokkos/pull/3845) +- Implement UniqueToken for SYCL [\#3833](https://github.com/kokkos/kokkos/pull/3833) +- OpenMPTarget: UniqueToken::Global implementation [\#3823](https://github.com/kokkos/kokkos/pull/3823) +- DualView sync's on ExecutionSpaces [\#3822](https://github.com/kokkos/kokkos/pull/3822) +- SYCL outer TeamPolicy parallel_reduce [\#3818](https://github.com/kokkos/kokkos/pull/3818) +- SYCL TeamPolicy::team_scan [\#3815](https://github.com/kokkos/kokkos/pull/3815) +- SYCL MDRangePolicy parallel_reduce [\#3801](https://github.com/kokkos/kokkos/pull/3801) +- Enable use of execution space instances in ScatterView [\#3786](https://github.com/kokkos/kokkos/pull/3786) +- SYCL TeamPolicy nested parallel_reduce [\#3783](https://github.com/kokkos/kokkos/pull/3783) +- OpenMPTarget: MDRange with TagType for parallel_for [\#3781](https://github.com/kokkos/kokkos/pull/3781) +- Adding OpenMPTarget parallel_scan [\#3655](https://github.com/kokkos/kokkos/pull/3655) +- SYCL basic TeamPolicy [\#3654](https://github.com/kokkos/kokkos/pull/3654) +- OpenMPTarget: scratch memory implementation [\#3611](https://github.com/kokkos/kokkos/pull/3611) + +**Implemented enhancements Backends and Archs:** +- SYCL choose a specific GPU [\#3918](https://github.com/kokkos/kokkos/pull/3918) +- [HIP] Lock access to scratch memory when using Teams [\#3916](https://github.com/kokkos/kokkos/pull/3916) +- [HIP] fix multithreaded access to get_next_driver [\#3908](https://github.com/kokkos/kokkos/pull/3908) +- Forward declare HIPHostPinnedSpace and SYCLSharedUSMSpace [\#3902](https://github.com/kokkos/kokkos/pull/3902) +- Let SYCL USMObjectMem use SharedAllocationRecord [\#3898](https://github.com/kokkos/kokkos/pull/3898) +- Implement clock_tic for SYCL [\#3893](https://github.com/kokkos/kokkos/pull/3893) +- Don't use a static variable in HIPInternal::scratch_space [\#3866](https://github.com/kokkos/kokkos/pull/3866)(https://github.com/kokkos/kokkos/pull/3866) +- Reuse memory for SYCL parallel_reduce [\#3873](https://github.com/kokkos/kokkos/pull/3873) +- Update SYCL compiler in CI [\#3826](https://github.com/kokkos/kokkos/pull/3826) +- Introduce HostSharedPtr to manage m_space_instance for Cuda/HIP/SYCL [\#3824](https://github.com/kokkos/kokkos/pull/3824) +- [HIP] Use shuffle for range reduction [\#3811](https://github.com/kokkos/kokkos/pull/3811) +- OpenMPTarget: Changes to the hierarchical parallelism [\#3808](https://github.com/kokkos/kokkos/pull/3808) +- Remove ExtendedReferenceWrapper for SYCL parallel_reduce [\#3802](https://github.com/kokkos/kokkos/pull/3802) +- Eliminate sycl_indirect_launch [\#3777](https://github.com/kokkos/kokkos/pull/3777) +- OpenMPTarget: scratch implementation for parallel_reduce [\#3776](https://github.com/kokkos/kokkos/pull/3776) +- Allow initializing SYCL execution space from sycl::queue and SYCL::impl_static_fence [\#3767](https://github.com/kokkos/kokkos/pull/3767) +- SYCL TeamPolicy scratch memory alternative [\#3763](https://github.com/kokkos/kokkos/pull/3763) +- Alternative implementation for SYCL TeamPolicy [\#3759](https://github.com/kokkos/kokkos/pull/3759) +- Unify handling of synchronous errors in SYCL [\#3754](https://github.com/kokkos/kokkos/pull/3754) +- core/Cuda: Half_t updates for cgsolve [\#3746](https://github.com/kokkos/kokkos/pull/3746) +- Unify HIPParallelLaunch structures [\#3733](https://github.com/kokkos/kokkos/pull/3733) +- Improve performance for SYCL parallel_reduce [\#3732](https://github.com/kokkos/kokkos/pull/3732) +- Use consistent types in Kokkos_OpenMPTarget_Parallel.hpp [\#3703](https://github.com/kokkos/kokkos/pull/3703) +- Implement non-blocking kernel launches for HIP backend [\#3697](https://github.com/kokkos/kokkos/pull/3697) +- Change SYCLInternal::m_queue std::unique_ptr -> std::optional [\#3677](https://github.com/kokkos/kokkos/pull/3677) +- Use alternative SYCL parallel_reduce implementation [\#3671](https://github.com/kokkos/kokkos/pull/3671) +- Use runtime values in KokkosExp_MDRangePolicy.hpp [\#3626](https://github.com/kokkos/kokkos/pull/3626) +- Clean up AnalyzePolicy [\#3564](https://github.com/kokkos/kokkos/pull/3564) +- Changes for indirect launch of SYCL parallel reduce [\#3511](https://github.com/kokkos/kokkos/pull/3511) + +**Implemented enhancements BuildSystem:** +- Also require C++14 when building gtest [\#3912](https://github.com/kokkos/kokkos/pull/3912) +- Fix compiling SYCL with OpenMP [\#3874](https://github.com/kokkos/kokkos/pull/3874) +- Require C++17 for SYCL (at configuration time) [\#3869](https://github.com/kokkos/kokkos/pull/3869) +- Add COMPILE_DEFINITIONS argument to kokkos_create_imported_tpl [\#3862](https://github.com/kokkos/kokkos/pull/3862) +- Do not pass arch flags to the linker with no rdc [\#3846](https://github.com/kokkos/kokkos/pull/3846) +- Try compiling C++14 check with C++14 support and print error message [\#3843](https://github.com/kokkos/kokkos/pull/3843) +- Enable HIP with Cray Clang [\#3842](https://github.com/kokkos/kokkos/pull/3842) +- Add an option to disable header self containment tests [\#3834](https://github.com/kokkos/kokkos/pull/3834) +- CMake check for C++14 [\#3809](https://github.com/kokkos/kokkos/pull/3809) +- Prefer -std=* over --std=* [\#3779](https://github.com/kokkos/kokkos/pull/3779) +- Kokkos launch compiler updates [\#3778](https://github.com/kokkos/kokkos/pull/3778) +- Updated comments and enabled no-op for kokkos_launch_compiler [\#3774](https://github.com/kokkos/kokkos/pull/3774) +- Apple's Clang not correctly recognised [\#3772](https://github.com/kokkos/kokkos/pull/3772) +- kokkos_launch_compiler + CUDA auto-detect arch [\#3770](https://github.com/kokkos/kokkos/pull/3770) +- Add Spack test support for Kokkos [\#3753](https://github.com/kokkos/kokkos/pull/3753) +- Split SYCL tests for aot compilation [\#3741](https://github.com/kokkos/kokkos/pull/3741) +- Use consistent OpenMP flag for IntelClang [\#3735](https://github.com/kokkos/kokkos/pull/3735) +- Add support for -Wno-deprecated-gpu-targets [\#3722](https://github.com/kokkos/kokkos/pull/3722) +- Add configuration to target CUDA compute capability 8.6 [\#3713](https://github.com/kokkos/kokkos/pull/3713) +- Added VERSION and SOVERSION to KOKKOS_INTERNAL_ADD_LIBRARY [\#3706](https://github.com/kokkos/kokkos/pull/3706) +- Add fast-math to known NVCC flags [\#3699](https://github.com/kokkos/kokkos/pull/3699) +- Add MI-100 arch string [\#3698](https://github.com/kokkos/kokkos/pull/3698) +- Require CMake >=3.16 [\#3679](https://github.com/kokkos/kokkos/pull/3679) +- KokkosCI.cmake, KokkosCTest.cmake.in, CTestConfig.cmake.in + CI updates [\#2844](https://github.com/kokkos/kokkos/pull/2844) + +**Implemented enhancements Tools:** +- Improve readability of the callback invocation in profiling [\#3860](https://github.com/kokkos/kokkos/pull/3860) +- V1.1 Tools Interface: incremental, action-based [\#3812](https://github.com/kokkos/kokkos/pull/3812) +- Enable launch latency simulations [\#3721](https://github.com/kokkos/kokkos/pull/3721) +- Added metadata callback to tools interface [\#3711](https://github.com/kokkos/kokkos/pull/3711) +- MDRange Tile Size Tuning [\#3688](https://github.com/kokkos/kokkos/pull/3688) +- Added support for command-line args for kokkos-tools [\#3627](https://github.com/kokkos/kokkos/pull/3627) +- Query max tile sizes for an MDRangePolicy, and set tile sizes on an existing policy [\#3481](https://github.com/kokkos/kokkos/pull/3481) + +**Implemented enhancements Other:** +- Try detecting ndevices in get_gpu [\#3921](https://github.com/kokkos/kokkos/pull/3921) +- Use strcmp to compare names() [\#3909](https://github.com/kokkos/kokkos/pull/3909) +- Add execution space arguments for constructor overloads that might allocate a new underlying View [\#3904](https://github.com/kokkos/kokkos/pull/3904) +- Prefix labels in internal use of kokkos_malloc [\#3891](https://github.com/kokkos/kokkos/pull/3891) +- Prefix labels for internal uses of SharedAllocationRecord [\#3890](https://github.com/kokkos/kokkos/pull/3890) +- Add missing hypot math function [\#3880](https://github.com/kokkos/kokkos/pull/3880) +- Unify algorithm unit tests to avoid code duplication [\#3851](https://github.com/kokkos/kokkos/pull/3851) +- DualView.template view() better matches for Devices in UVMSpace cases [\#3857](https://github.com/kokkos/kokkos/pull/3857) +- More extensive disentangling of Policy Traits [\#3829](https://github.com/kokkos/kokkos/pull/3829) +- Replaced nanosleep and sched_yield with STL routines [\#3825](https://github.com/kokkos/kokkos/pull/3825) +- Constructing Atomic Subviews [\#3810](https://github.com/kokkos/kokkos/pull/3810) +- Metadata Declaration in Core [\#3729](https://github.com/kokkos/kokkos/pull/3729) +- Allow using tagged final functor in parallel_reduce [\#3714](https://github.com/kokkos/kokkos/pull/3714) +- Major duplicate code removal in SharedAllocationRecord specializations [\#3658](https://github.com/kokkos/kokkos/pull/3658) + +**Fixed bugs:** +- Provide forward declarations in Kokkos_ViewLayoutTiled.hpp for XL [\#3911](https://github.com/kokkos/kokkos/pull/3911) +- Fixup absolute value of floating points in Kokkos complex [\#3882](https://github.com/kokkos/kokkos/pull/3882) +- Address intel 17 ICE [\#3881](https://github.com/kokkos/kokkos/pull/3881) +- Add missing pow(Kokkos::complex) overloads [\#3868](https://github.com/kokkos/kokkos/pull/3868) +- Fix bug {pow, log}(Kokkos::complex) [\#3866](https://github.com/kokkos/kokkos/pull/3866)(https://github.com/kokkos/kokkos/pull/3866) +- Cleanup writing to output streams in Cuda [\#3859](https://github.com/kokkos/kokkos/pull/3859) +- Fixup cache CUDA fallback execution space instance used by DualView::sync [\#3856](https://github.com/kokkos/kokkos/pull/3856) +- Fix cmake warning with pthread [\#3854](https://github.com/kokkos/kokkos/pull/3854) +- Fix typo FOUND_CUDA_{DRIVVER -> DRIVER} [\#3852](https://github.com/kokkos/kokkos/pull/3852) +- Fix bug in SYCL team_reduce [\#3848](https://github.com/kokkos/kokkos/pull/3848) +- Atrocious bug in MDRange tuning [\#3803](https://github.com/kokkos/kokkos/pull/3803) +- Fix compiling SYCL with Kokkos_ENABLE_TUNING=ON [\#3800](https://github.com/kokkos/kokkos/pull/3800) +- Fixed command line parsing bug [\#3797](https://github.com/kokkos/kokkos/pull/3797) +- Workaround race condition in SYCL parallel_reduce [\#3782](https://github.com/kokkos/kokkos/pull/3782) +- Fix Atomic{Min,Max} for Kepler30 [\#3780](https://github.com/kokkos/kokkos/pull/3780) +- Fix SYCL typo [\#3755](https://github.com/kokkos/kokkos/pull/3755) +- Fixed Kokkos_install_additional_files macro [\#3752](https://github.com/kokkos/kokkos/pull/3752) +- Fix a typo for Kokkos_ARCH_A64FX [\#3751](https://github.com/kokkos/kokkos/pull/3751) +- OpenMPTarget: fixes and workarounds to work with "Release" build type [\#3748](https://github.com/kokkos/kokkos/pull/3748) +- Fix parsing bug for number of devices command line argument [\#3724](https://github.com/kokkos/kokkos/pull/3724) +- Avoid more warnings with clang and C++20 [\#3719](https://github.com/kokkos/kokkos/pull/3719) +- Fix gcc-10.1 C++20 warnings [\#3718](https://github.com/kokkos/kokkos/pull/3718) +- Fix cuda cache config not being set correct [\#3712](https://github.com/kokkos/kokkos/pull/3712) +- Fix dualview deepcopy perftools [\#3701](https://github.com/kokkos/kokkos/pull/3701) +- use drand instead of frand in drand [\#3696](https://github.com/kokkos/kokkos/pull/3696) + +**Incompatibilities:** +- Remove unimplemented member functions of SYCLDevice [\#3919](https://github.com/kokkos/kokkos/pull/3919) +- Replace cl::sycl [\#3896](https://github.com/kokkos/kokkos/pull/3896) +- Get rid of SYCL workaround in Kokkos_Complex.hpp [\#3884](https://github.com/kokkos/kokkos/pull/3884) +- Replace most uses of if_c [\#3883](https://github.com/kokkos/kokkos/pull/3883) +- Remove Impl::enable_if_type [\#3863](https://github.com/kokkos/kokkos/pull/3863) +- Remove HostBarrier test [\#3847](https://github.com/kokkos/kokkos/pull/3847) +- Avoid (void) interface [\#3836](https://github.com/kokkos/kokkos/pull/3836) +- Remove VerifyExecutionCanAccessMemorySpace [\#3813](https://github.com/kokkos/kokkos/pull/3813) +- Avoid duplicated code in ScratchMemorySpace [\#3793](https://github.com/kokkos/kokkos/pull/3793) +- Remove superfluous FunctorFinal specialization [\#3788](https://github.com/kokkos/kokkos/pull/3788) +- Rename cl::sycl -> sycl in Kokkos_MathematicalFunctions.hpp [\#3678](https://github.com/kokkos/kokkos/pull/3678) +- Remove integer_sequence backward compatibility implementation [\#3533](https://github.com/kokkos/kokkos/pull/3533) + +**Enabled tests:** +- Fixup re-enable core performance tests [\#3903](https://github.com/kokkos/kokkos/pull/3903) +- Enable more SYCL tests [\#3900](https://github.com/kokkos/kokkos/pull/3900) +- Restrict MDRange Policy tests for Intel GPUs [\#3853](https://github.com/kokkos/kokkos/pull/3853) +- Disable death tests for rawhide [\#3844](https://github.com/kokkos/kokkos/pull/3844) +- OpenMPTarget: Block unit tests that do not pass with the nvidia compiler [\#3839](https://github.com/kokkos/kokkos/pull/3839) +- Enable Bitset container test for SYCL [\#3830](https://github.com/kokkos/kokkos/pull/3830) +- Enable some more SYCL tests [\#3744](https://github.com/kokkos/kokkos/pull/3744) +- Enable SYCL atomic tests [\#3742](https://github.com/kokkos/kokkos/pull/3742) +- Enable more SYCL perf_tests [\#3692](https://github.com/kokkos/kokkos/pull/3692) +- Enable examples for SYCL [\#3691](https://github.com/kokkos/kokkos/pull/3691) + ## [3.3.01](https://github.com/kokkos/kokkos/tree/3.3.01) (2021-01-06) [Full Changelog](https://github.com/kokkos/kokkos/compare/3.3.00...3.3.01) diff --git a/lib/kokkos/CMakeLists.txt b/lib/kokkos/CMakeLists.txt index 7bc3c77256..6fc1bf7d2f 100644 --- a/lib/kokkos/CMakeLists.txt +++ b/lib/kokkos/CMakeLists.txt @@ -72,7 +72,7 @@ ENDFUNCTION() LIST(APPEND CMAKE_MODULE_PATH cmake/Modules) IF(NOT KOKKOS_HAS_TRILINOS) - cmake_minimum_required(VERSION 3.10 FATAL_ERROR) + cmake_minimum_required(VERSION 3.16 FATAL_ERROR) set(CMAKE_DISABLE_SOURCE_CHANGES ON) set(CMAKE_DISABLE_IN_SOURCE_BUILD ON) IF (Spack_WORKAROUND) @@ -111,27 +111,25 @@ ENDIF() set(Kokkos_VERSION_MAJOR 3) -set(Kokkos_VERSION_MINOR 3) -set(Kokkos_VERSION_PATCH 1) +set(Kokkos_VERSION_MINOR 4) +set(Kokkos_VERSION_PATCH 00) set(Kokkos_VERSION "${Kokkos_VERSION_MAJOR}.${Kokkos_VERSION_MINOR}.${Kokkos_VERSION_PATCH}") math(EXPR KOKKOS_VERSION "${Kokkos_VERSION_MAJOR} * 10000 + ${Kokkos_VERSION_MINOR} * 100 + ${Kokkos_VERSION_PATCH}") -IF(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.12.0") - MESSAGE(STATUS "Setting policy CMP0074 to use _ROOT variables") - CMAKE_POLICY(SET CMP0074 NEW) -ENDIF() +MESSAGE(STATUS "Setting policy CMP0074 to use _ROOT variables") +CMAKE_POLICY(SET CMP0074 NEW) # Load either the real TriBITS or a TriBITS wrapper # for certain utility functions that are universal (like GLOBAL_SET) INCLUDE(${KOKKOS_SRC_PATH}/cmake/fake_tribits.cmake) -IF (Kokkos_ENABLE_CUDA AND ${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.14.0") - #If we are building CUDA, we have tricked CMake because we declare a CXX project - #If the default C++ standard for a given compiler matches the requested - #standard, then CMake just omits the -std flag in later versions of CMake - #This breaks CUDA compilation (CUDA compiler can have a different default - #-std then the underlying host compiler by itself). Setting this variable - #forces CMake to always add the -std flag even if it thinks it doesn't need it +IF (Kokkos_ENABLE_CUDA) + # If we are building CUDA, we have tricked CMake because we declare a CXX project + # If the default C++ standard for a given compiler matches the requested + # standard, then CMake just omits the -std flag in later versions of CMake + # This breaks CUDA compilation (CUDA compiler can have a different default + # -std then the underlying host compiler by itself). Setting this variable + # forces CMake to always add the -std flag even if it thinks it doesn't need it GLOBAL_SET(CMAKE_CXX_STANDARD_DEFAULT 98) ENDIF() @@ -139,15 +137,19 @@ ENDIF() # I really wish these were regular variables # but scoping issues can make it difficult GLOBAL_SET(KOKKOS_COMPILE_OPTIONS) -GLOBAL_SET(KOKKOS_LINK_OPTIONS -DKOKKOS_DEPENDENCE) +GLOBAL_SET(KOKKOS_LINK_OPTIONS) GLOBAL_SET(KOKKOS_CUDA_OPTIONS) GLOBAL_SET(KOKKOS_CUDAFE_OPTIONS) GLOBAL_SET(KOKKOS_XCOMPILER_OPTIONS) # We need to append text here for making sure TPLs # we import are available for an installed Kokkos GLOBAL_SET(KOKKOS_TPL_EXPORTS) -# this could probably be scoped to project +# KOKKOS_DEPENDENCE is used by kokkos_launch_compiler GLOBAL_SET(KOKKOS_COMPILE_DEFINITIONS KOKKOS_DEPENDENCE) +# MSVC never goes through kokkos_launch_compiler +IF(NOT MSVC) + GLOBAL_APPEND(KOKKOS_LINK_OPTIONS -DKOKKOS_DEPENDENCE) +ENDIF() # Include a set of Kokkos-specific wrapper functions that # will either call raw CMake or TriBITS diff --git a/lib/kokkos/Makefile.kokkos b/lib/kokkos/Makefile.kokkos index 061b7a46ee..aa97f99b75 100644 --- a/lib/kokkos/Makefile.kokkos +++ b/lib/kokkos/Makefile.kokkos @@ -11,8 +11,8 @@ CXXFLAGS += $(SHFLAGS) endif KOKKOS_VERSION_MAJOR = 3 -KOKKOS_VERSION_MINOR = 3 -KOKKOS_VERSION_PATCH = 1 +KOKKOS_VERSION_MINOR = 4 +KOKKOS_VERSION_PATCH = 00 KOKKOS_VERSION = $(shell echo $(KOKKOS_VERSION_MAJOR)*10000+$(KOKKOS_VERSION_MINOR)*100+$(KOKKOS_VERSION_PATCH) | bc) # Options: Cuda,HIP,OpenMP,Pthread,Serial @@ -20,7 +20,7 @@ KOKKOS_DEVICES ?= "OpenMP" #KOKKOS_DEVICES ?= "Pthread" # Options: # Intel: KNC,KNL,SNB,HSW,BDW,SKX -# NVIDIA: Kepler,Kepler30,Kepler32,Kepler35,Kepler37,Maxwell,Maxwell50,Maxwell52,Maxwell53,Pascal60,Pascal61,Volta70,Volta72,Turing75,Ampere80 +# NVIDIA: Kepler,Kepler30,Kepler32,Kepler35,Kepler37,Maxwell,Maxwell50,Maxwell52,Maxwell53,Pascal60,Pascal61,Volta70,Volta72,Turing75,Ampere80,Ampere86 # ARM: ARMv80,ARMv81,ARMv8-ThunderX,ARMv8-TX2,A64FX # IBM: BGQ,Power7,Power8,Power9 # AMD-GPUS: Vega900,Vega906,Vega908 @@ -164,17 +164,17 @@ KOKKOS_INTERNAL_OS_DARWIN := $(call kokkos_has_string,$(KOKKOS_OS),Darwin) KOKKOS_CXX_VERSION := $(strip $(shell $(CXX) --version 2>&1)) KOKKOS_INTERNAL_COMPILER_INTEL := $(call kokkos_has_string,$(KOKKOS_CXX_VERSION),Intel Corporation) KOKKOS_INTERNAL_COMPILER_PGI := $(call kokkos_has_string,$(KOKKOS_CXX_VERSION),PGI) -KOKKOS_INTERNAL_COMPILER_XL := $(strip $(shell $(CXX) -qversion 2>&1 | grep XL | wc -l)) -KOKKOS_INTERNAL_COMPILER_CRAY := $(strip $(shell $(CXX) -craype-verbose 2>&1 | grep "CC-" | wc -l)) -KOKKOS_INTERNAL_COMPILER_NVCC := $(strip $(shell echo "$(shell export OMPI_CXX=$(OMPI_CXX); export MPICH_CXX=$(MPICH_CXX); $(CXX) --version 2>&1 | grep nvcc | wc -l)>0" | bc)) +KOKKOS_INTERNAL_COMPILER_XL := $(strip $(shell $(CXX) -qversion 2>&1 | grep -c XL)) +KOKKOS_INTERNAL_COMPILER_CRAY := $(strip $(shell $(CXX) -craype-verbose 2>&1 | grep -c "CC-")) +KOKKOS_INTERNAL_COMPILER_NVCC := $(strip $(shell echo "$(shell export OMPI_CXX=$(OMPI_CXX); export MPICH_CXX=$(MPICH_CXX); $(CXX) --version 2>&1 | grep -c nvcc)>0" | bc)) KOKKOS_INTERNAL_COMPILER_CLANG := $(call kokkos_has_string,$(KOKKOS_CXX_VERSION),clang) -KOKKOS_INTERNAL_COMPILER_APPLE_CLANG := $(call kokkos_has_string,$(KOKKOS_CXX_VERSION),Apple LLVM) +KOKKOS_INTERNAL_COMPILER_APPLE_CLANG := $(call kokkos_has_string,$(KOKKOS_CXX_VERSION),Apple clang) KOKKOS_INTERNAL_COMPILER_HCC := $(call kokkos_has_string,$(KOKKOS_CXX_VERSION),HCC) KOKKOS_INTERNAL_COMPILER_GCC := $(call kokkos_has_string,$(KOKKOS_CXX_VERSION),GCC) # Check Host Compiler if using NVCC through nvcc_wrapper ifeq ($(KOKKOS_INTERNAL_COMPILER_NVCC), 1) - KOKKOS_INTERNAL_COMPILER_NVCC_WRAPPER := $(strip $(shell echo $(CXX) | grep nvcc_wrapper | wc -l)) + KOKKOS_INTERNAL_COMPILER_NVCC_WRAPPER := $(strip $(shell echo $(CXX) | grep -c nvcc_wrapper)) ifeq ($(KOKKOS_INTERNAL_COMPILER_NVCC_WRAPPER), 1) KOKKOS_CXX_HOST_VERSION := $(strip $(shell $(CXX) $(CXXFLAGS) --host-version 2>&1)) @@ -297,11 +297,11 @@ else #KOKKOS_INTERNAL_CXX1Z_FLAG := -hstd=c++1z #KOKKOS_INTERNAL_CXX2A_FLAG := -hstd=c++2a else - KOKKOS_INTERNAL_CXX14_FLAG := --std=c++14 - KOKKOS_INTERNAL_CXX1Y_FLAG := --std=c++1y - KOKKOS_INTERNAL_CXX17_FLAG := --std=c++17 - KOKKOS_INTERNAL_CXX1Z_FLAG := --std=c++1z - KOKKOS_INTERNAL_CXX2A_FLAG := --std=c++2a + KOKKOS_INTERNAL_CXX14_FLAG := -std=c++14 + KOKKOS_INTERNAL_CXX1Y_FLAG := -std=c++1y + KOKKOS_INTERNAL_CXX17_FLAG := -std=c++17 + KOKKOS_INTERNAL_CXX1Z_FLAG := -std=c++1z + KOKKOS_INTERNAL_CXX2A_FLAG := -std=c++2a endif endif endif @@ -332,6 +332,7 @@ KOKKOS_INTERNAL_USE_ARCH_VOLTA70 := $(call kokkos_has_string,$(KOKKOS_ARCH),Volt KOKKOS_INTERNAL_USE_ARCH_VOLTA72 := $(call kokkos_has_string,$(KOKKOS_ARCH),Volta72) KOKKOS_INTERNAL_USE_ARCH_TURING75 := $(call kokkos_has_string,$(KOKKOS_ARCH),Turing75) KOKKOS_INTERNAL_USE_ARCH_AMPERE80 := $(call kokkos_has_string,$(KOKKOS_ARCH),Ampere80) +KOKKOS_INTERNAL_USE_ARCH_AMPERE86 := $(call kokkos_has_string,$(KOKKOS_ARCH),Ampere86) KOKKOS_INTERNAL_USE_ARCH_NVIDIA := $(shell expr $(KOKKOS_INTERNAL_USE_ARCH_KEPLER30) \ + $(KOKKOS_INTERNAL_USE_ARCH_KEPLER32) \ + $(KOKKOS_INTERNAL_USE_ARCH_KEPLER35) \ @@ -344,7 +345,8 @@ KOKKOS_INTERNAL_USE_ARCH_NVIDIA := $(shell expr $(KOKKOS_INTERNAL_USE_ARCH_KEPLE + $(KOKKOS_INTERNAL_USE_ARCH_VOLTA70) \ + $(KOKKOS_INTERNAL_USE_ARCH_VOLTA72) \ + $(KOKKOS_INTERNAL_USE_ARCH_TURING75) \ - + $(KOKKOS_INTERNAL_USE_ARCH_AMPERE80)) + + $(KOKKOS_INTERNAL_USE_ARCH_AMPERE80) \ + + $(KOKKOS_INTERNAL_USE_ARCH_AMPERE86)) #SEK: This seems like a bug to me ifeq ($(KOKKOS_INTERNAL_USE_ARCH_NVIDIA), 0) @@ -585,10 +587,10 @@ ifeq ($(KOKKOS_INTERNAL_ENABLE_PROFILING_LOAD_PRINT), 1) endif ifeq ($(KOKKOS_INTERNAL_ENABLE_TUNING), 1) - tmp := $(call kokkos_append_header,"\#define KOKKOS_ENABLE_TUNING") + tmp := $(call kokkos_append_header,"$H""define KOKKOS_ENABLE_TUNING") endif -tmp := $(call kokkos_append_header,"\#define KOKKOS_ENABLE_LIBDL") +tmp := $(call kokkos_append_header,"$H""define KOKKOS_ENABLE_LIBDL") ifeq ($(KOKKOS_INTERNAL_USE_HWLOC), 1) ifneq ($(KOKKOS_CMAKE), yes) @@ -752,6 +754,14 @@ ifeq ($(KOKKOS_INTERNAL_USE_ARCH_A64FX), 1) KOKKOS_CXXFLAGS += -march=armv8.2-a+sve KOKKOS_LDFLAGS += -march=armv8.2-a+sve + ifeq ($(KOKKOS_INTERNAL_COMPILER_CLANG), 1) + KOKKOS_CXXFLAGS += -msve-vector-bits=512 + KOKKOS_LDFLAGS += -msve-vector-bits=512 + endif + ifeq ($(KOKKOS_INTERNAL_COMPILER_GCC), 1) + KOKKOS_CXXFLAGS += -msve-vector-bits=512 + KOKKOS_LDFLAGS += -msve-vector-bits=512 + endif endif ifeq ($(KOKKOS_INTERNAL_USE_ARCH_ZEN), 1) @@ -1100,6 +1110,11 @@ ifeq ($(KOKKOS_INTERNAL_USE_CUDA_ARCH), 1) tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AMPERE80") KOKKOS_INTERNAL_CUDA_ARCH_FLAG := $(KOKKOS_INTERNAL_CUDA_ARCH_FLAG)=sm_80 endif + ifeq ($(KOKKOS_INTERNAL_USE_ARCH_AMPERE86), 1) + tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AMPERE") + tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AMPERE86") + KOKKOS_INTERNAL_CUDA_ARCH_FLAG := $(KOKKOS_INTERNAL_CUDA_ARCH_FLAG)=sm_86 + endif ifneq ($(KOKKOS_INTERNAL_USE_ARCH_NVIDIA), 0) KOKKOS_CXXFLAGS += $(KOKKOS_INTERNAL_CUDA_ARCH_FLAG) @@ -1159,7 +1174,7 @@ endif KOKKOS_INTERNAL_LS_CONFIG := $(shell ls KokkosCore_config.h 2>&1) ifeq ($(KOKKOS_INTERNAL_LS_CONFIG), KokkosCore_config.h) - KOKKOS_INTERNAL_NEW_CONFIG := $(strip $(shell diff KokkosCore_config.h KokkosCore_config.tmp | grep define | wc -l)) + KOKKOS_INTERNAL_NEW_CONFIG := $(strip $(shell diff KokkosCore_config.h KokkosCore_config.tmp | grep -c define)) else KOKKOS_INTERNAL_NEW_CONFIG := 1 endif @@ -1181,41 +1196,41 @@ tmp := $(call kokkos_update_config_header, KOKKOS_SETUP_HPP_, "KokkosCore_Config tmp := $(call kokkos_update_config_header, KOKKOS_DECLARE_HPP_, "KokkosCore_Config_DeclareBackend.tmp", "KokkosCore_Config_DeclareBackend.hpp") tmp := $(call kokkos_update_config_header, KOKKOS_POST_INCLUDE_HPP_, "KokkosCore_Config_PostInclude.tmp", "KokkosCore_Config_PostInclude.hpp") ifeq ($(KOKKOS_INTERNAL_USE_CUDA), 1) - tmp := $(call kokkos_append_config_header,"\#include ","KokkosCore_Config_FwdBackend.hpp") - tmp := $(call kokkos_append_config_header,"\#include ","KokkosCore_Config_DeclareBackend.hpp") - tmp := $(call kokkos_append_config_header,"\#include ","KokkosCore_Config_SetupBackend.hpp") + tmp := $(call kokkos_append_config_header,"$H""include ","KokkosCore_Config_FwdBackend.hpp") + tmp := $(call kokkos_append_config_header,"$H""include ","KokkosCore_Config_DeclareBackend.hpp") + tmp := $(call kokkos_append_config_header,"$H""include ","KokkosCore_Config_SetupBackend.hpp") ifeq ($(KOKKOS_INTERNAL_CUDA_USE_UVM), 1) else endif endif ifeq ($(KOKKOS_INTERNAL_USE_OPENMPTARGET), 1) - tmp := $(call kokkos_append_config_header,"\#include ","KokkosCore_Config_FwdBackend.hpp") - tmp := $(call kokkos_append_config_header,"\#include ","KokkosCore_Config_DeclareBackend.hpp") + tmp := $(call kokkos_append_config_header,"$H""include ","KokkosCore_Config_FwdBackend.hpp") + tmp := $(call kokkos_append_config_header,"$H""include ","KokkosCore_Config_DeclareBackend.hpp") endif ifeq ($(KOKKOS_INTERNAL_USE_HIP), 1) - tmp := $(call kokkos_append_config_header,"\#include ","KokkosCore_Config_FwdBackend.hpp") - tmp := $(call kokkos_append_config_header,"\#include ","KokkosCore_Config_DeclareBackend.hpp") - tmp := $(call kokkos_append_config_header,"\#include ","KokkosCore_Config_SetupBackend.hpp") + tmp := $(call kokkos_append_config_header,"$H""include ","KokkosCore_Config_FwdBackend.hpp") + tmp := $(call kokkos_append_config_header,"$H""include ","KokkosCore_Config_DeclareBackend.hpp") + tmp := $(call kokkos_append_config_header,"$H""include ","KokkosCore_Config_SetupBackend.hpp") endif ifeq ($(KOKKOS_INTERNAL_USE_OPENMP), 1) - tmp := $(call kokkos_append_config_header,"\#include ","KokkosCore_Config_FwdBackend.hpp") - tmp := $(call kokkos_append_config_header,"\#include ","KokkosCore_Config_DeclareBackend.hpp") + tmp := $(call kokkos_append_config_header,"$H""include ","KokkosCore_Config_FwdBackend.hpp") + tmp := $(call kokkos_append_config_header,"$H""include ","KokkosCore_Config_DeclareBackend.hpp") endif ifeq ($(KOKKOS_INTERNAL_USE_PTHREADS), 1) - tmp := $(call kokkos_append_config_header,"\#include ","KokkosCore_Config_FwdBackend.hpp") - tmp := $(call kokkos_append_config_header,"\#include ","KokkosCore_Config_DeclareBackend.hpp") + tmp := $(call kokkos_append_config_header,"$H""include ","KokkosCore_Config_FwdBackend.hpp") + tmp := $(call kokkos_append_config_header,"$H""include ","KokkosCore_Config_DeclareBackend.hpp") endif ifeq ($(KOKKOS_INTERNAL_USE_HPX), 1) - tmp := $(call kokkos_append_config_header,"\#include ","KokkosCore_Config_FwdBackend.hpp") - tmp := $(call kokkos_append_config_header,"\#include ","KokkosCore_Config_DeclareBackend.hpp") + tmp := $(call kokkos_append_config_header,"$H""include ","KokkosCore_Config_FwdBackend.hpp") + tmp := $(call kokkos_append_config_header,"$H""include ","KokkosCore_Config_DeclareBackend.hpp") endif ifeq ($(KOKKOS_INTERNAL_USE_SERIAL), 1) - tmp := $(call kokkos_append_config_header,"\#include ","KokkosCore_Config_FwdBackend.hpp") - tmp := $(call kokkos_append_config_header,"\#include ","KokkosCore_Config_DeclareBackend.hpp") + tmp := $(call kokkos_append_config_header,"$H""include ","KokkosCore_Config_FwdBackend.hpp") + tmp := $(call kokkos_append_config_header,"$H""include ","KokkosCore_Config_DeclareBackend.hpp") endif ifeq ($(KOKKOS_INTERNAL_USE_MEMKIND), 1) - tmp := $(call kokkos_append_config_header,"\#include ","KokkosCore_Config_FwdBackend.hpp") - tmp := $(call kokkos_append_config_header,"\#include ","KokkosCore_Config_DeclareBackend.hpp") + tmp := $(call kokkos_append_config_header,"$H""include ","KokkosCore_Config_FwdBackend.hpp") + tmp := $(call kokkos_append_config_header,"$H""include ","KokkosCore_Config_DeclareBackend.hpp") endif KOKKOS_HEADERS += $(wildcard $(KOKKOS_PATH)/core/src/*.hpp) KOKKOS_HEADERS += $(wildcard $(KOKKOS_PATH)/core/src/impl/*.hpp) @@ -1334,7 +1349,7 @@ ifneq ($(KOKKOS_INTERNAL_USE_SERIAL), 1) endif # With Cygwin functions such as fdopen and fileno are not defined -# when strict ansi is enabled. strict ansi gets enabled with --std=c++14 +# when strict ansi is enabled. strict ansi gets enabled with -std=c++14 # though. So we hard undefine it here. Not sure if that has any bad side effects # This is needed for gtest actually, not for Kokkos itself! ifeq ($(KOKKOS_INTERNAL_OS_CYGWIN), 1) diff --git a/lib/kokkos/Makefile.targets b/lib/kokkos/Makefile.targets index 5a03f7d17e..cf9fc24242 100644 --- a/lib/kokkos/Makefile.targets +++ b/lib/kokkos/Makefile.targets @@ -36,6 +36,8 @@ Kokkos_MemorySpace.o: $(KOKKOS_CPP_DEPENDS) $(KOKKOS_PATH)/core/src/impl/Kokkos_ $(CXX) $(KOKKOS_CPPFLAGS) $(KOKKOS_CXXFLAGS) $(CXXFLAGS) -c $(KOKKOS_PATH)/core/src/impl/Kokkos_MemorySpace.cpp Kokkos_HostSpace_deepcopy.o: $(KOKKOS_CPP_DEPENDS) $(KOKKOS_PATH)/core/src/impl/Kokkos_HostSpace_deepcopy.cpp $(CXX) $(KOKKOS_CPPFLAGS) $(KOKKOS_CXXFLAGS) $(CXXFLAGS) -c $(KOKKOS_PATH)/core/src/impl/Kokkos_HostSpace_deepcopy.cpp +Kokkos_NumericTraits.o: $(KOKKOS_CPP_DEPENDS) $(KOKKOS_PATH)/core/src/impl/Kokkos_NumericTraits.cpp + $(CXX) $(KOKKOS_CPPFLAGS) $(KOKKOS_CXXFLAGS) $(CXXFLAGS) -c $(KOKKOS_PATH)/core/src/impl/Kokkos_NumericTraits.cpp ifeq ($(KOKKOS_INTERNAL_USE_CUDA), 1) Kokkos_Cuda_Instance.o: $(KOKKOS_CPP_DEPENDS) $(KOKKOS_PATH)/core/src/Cuda/Kokkos_Cuda_Instance.cpp diff --git a/lib/kokkos/algorithms/src/Kokkos_Random.hpp b/lib/kokkos/algorithms/src/Kokkos_Random.hpp index 69d6cf8f35..904cf5ccb9 100644 --- a/lib/kokkos/algorithms/src/Kokkos_Random.hpp +++ b/lib/kokkos/algorithms/src/Kokkos_Random.hpp @@ -668,6 +668,25 @@ struct Random_UniqueIndex { }; #endif +#ifdef KOKKOS_ENABLE_SYCL +template <> +struct Random_UniqueIndex { + using locks_view_type = View; + KOKKOS_FUNCTION + static int get_state_idx(const locks_view_type& locks_) { +#ifdef KOKKOS_ARCH_INTEL_GEN + int i = Kokkos::Impl::clock_tic() % locks_.extent(0); +#else + int i = 0; +#endif + while (Kokkos::atomic_compare_exchange(&locks_(i), 0, 1)) { + i = (i + 1) % static_cast(locks_.extent(0)); + } + return i; + } +}; +#endif + } // namespace Impl template @@ -1028,7 +1047,7 @@ class Random_XorShift1024 { KOKKOS_INLINE_FUNCTION double drand(const double& start, const double& end) { - return frand(end - start) + start; + return drand(end - start) + start; } // Marsaglia polar method for drawing a standard normal distributed random diff --git a/lib/kokkos/algorithms/unit_tests/CMakeLists.txt b/lib/kokkos/algorithms/unit_tests/CMakeLists.txt index 819c9e54ba..9109837985 100644 --- a/lib/kokkos/algorithms/unit_tests/CMakeLists.txt +++ b/lib/kokkos/algorithms/unit_tests/CMakeLists.txt @@ -3,6 +3,7 @@ KOKKOS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) KOKKOS_INCLUDE_DIRECTORIES(REQUIRED_DURING_INSTALLATION_TESTING ${CMAKE_CURRENT_SOURCE_DIR}) KOKKOS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../src ) +KOKKOS_INCLUDE_DIRECTORIES(${KOKKOS_SOURCE_DIR}/core/unit_test/category_files) SET(GTEST_SOURCE_DIR ${${PARENT_PACKAGE_NAME}_SOURCE_DIR}/tpls/gtest) @@ -25,7 +26,7 @@ KOKKOS_ADD_TEST_LIBRARY( TARGET_COMPILE_DEFINITIONS(kokkosalgorithms_gtest PUBLIC GTEST_HAS_TR1_TUPLE=0 GTEST_HAS_PTHREAD=0) IF((NOT (Kokkos_ENABLE_CUDA AND WIN32)) AND (NOT ("${KOKKOS_CXX_COMPILER_ID}" STREQUAL "Fujitsu"))) -TARGET_COMPILE_FEATURES(kokkosalgorithms_gtest PUBLIC cxx_std_11) + TARGET_COMPILE_FEATURES(kokkosalgorithms_gtest PUBLIC cxx_std_14) ENDIF() # Suppress clang-tidy diagnostics on code that we do not have control over @@ -33,51 +34,42 @@ IF(CMAKE_CXX_CLANG_TIDY) SET_TARGET_PROPERTIES(kokkosalgorithms_gtest PROPERTIES CXX_CLANG_TIDY "") ENDIF() -SET(SOURCES - UnitTestMain.cpp -) +SET(ALGORITHM UnitTestMain.cpp) IF(Kokkos_ENABLE_OPENMP) - LIST( APPEND SOURCES - TestOpenMP.cpp + LIST(APPEND ALGORITHM_SOURCES TestOpenMP_Sort1D.cpp TestOpenMP_Sort3D.cpp TestOpenMP_SortDynamicView.cpp - TestOpenMP_Random.cpp ) ENDIF() -IF(Kokkos_ENABLE_HIP) - LIST( APPEND SOURCES - TestHIP.cpp - ) -ENDIF() +foreach(Tag Threads;Serial;OpenMP;Cuda;HPX;HIP;SYCL) + # Because there is always an exception to the rule + if(Tag STREQUAL "Threads") + set(DEVICE "PTHREAD") + else() + string(TOUPPER ${Tag} DEVICE) + endif() -IF(Kokkos_ENABLE_CUDA) - LIST( APPEND SOURCES - TestCuda.cpp - ) -ENDIF() - -IF(Kokkos_ENABLE_HPX) - LIST( APPEND SOURCES - TestHPX.cpp - ) -ENDIF() - -IF(Kokkos_ENABLE_SERIAL) - LIST( APPEND SOURCES - TestSerial.cpp - ) -ENDIF() - -IF(Kokkos_ENABLE_PTHREAD) - LIST( APPEND SOURCES - TestThreads.cpp - ) -ENDIF() + if(Kokkos_ENABLE_${DEVICE}) + set(dir ${CMAKE_CURRENT_BINARY_DIR}) + set(file ${dir}/Test${Tag}.cpp) + # Write to a temporary intermediate file and call configure_file to avoid + # updating timestamps triggering unnecessary rebuilds on subsequent cmake runs. + file(WRITE ${dir}/dummy.cpp + "#include \n" + "#include \n" + "#include \n" + ) + configure_file(${dir}/dummy.cpp ${file}) + list(APPEND ALGORITHM_SOURCES ${file}) + endif() +endforeach() KOKKOS_ADD_EXECUTABLE_AND_TEST( UnitTest - SOURCES ${SOURCES} + SOURCES + UnitTestMain.cpp + ${ALGORITHM_SOURCES} ) diff --git a/lib/kokkos/algorithms/unit_tests/Makefile b/lib/kokkos/algorithms/unit_tests/Makefile index c112d7c6fc..dd0aa87de0 100644 --- a/lib/kokkos/algorithms/unit_tests/Makefile +++ b/lib/kokkos/algorithms/unit_tests/Makefile @@ -20,11 +20,19 @@ override LDFLAGS += -lpthread include $(KOKKOS_PATH)/Makefile.kokkos -KOKKOS_CXXFLAGS += -I$(GTEST_PATH) -I${KOKKOS_PATH}/algorithms/unit_tests +KOKKOS_CXXFLAGS += -I$(GTEST_PATH) -I${KOKKOS_PATH}/algorithms/unit_tests -I${KOKKOS_PATH}/core/unit_test/category_files TEST_TARGETS = TARGETS = +tmp := $(foreach device, $(KOKKOS_DEVICELIST), \ + $(if $(filter Test$(device).cpp, $(shell ls Test$(device).cpp 2>/dev/null)),,\ + $(shell echo "\#include " > Test$(device).cpp); \ + $(shell echo "\#include " >> Test$(device).cpp); \ + $(shell echo "\#include " >> Test$(device).cpp); \ + ) \ +) + ifeq ($(KOKKOS_INTERNAL_USE_CUDA), 1) OBJ_CUDA = TestCuda.o UnitTestMain.o gtest-all.o TARGETS += KokkosAlgorithms_UnitTest_Cuda @@ -44,7 +52,7 @@ ifeq ($(KOKKOS_INTERNAL_USE_PTHREADS), 1) endif ifeq ($(KOKKOS_INTERNAL_USE_OPENMP), 1) - OBJ_OPENMP = TestOpenMP.o TestOpenMP_Random.o TestOpenMP_Sort1D.o TestOpenMP_Sort3D.o TestOpenMP_SortDynamicView.o UnitTestMain.o gtest-all.o + OBJ_OPENMP = TestOpenMP.o TestOpenMP_Sort1D.o TestOpenMP_Sort3D.o TestOpenMP_SortDynamicView.o UnitTestMain.o gtest-all.o TARGETS += KokkosAlgorithms_UnitTest_OpenMP TEST_TARGETS += test-openmp endif diff --git a/lib/kokkos/algorithms/unit_tests/TestOpenMP_Sort1D.cpp b/lib/kokkos/algorithms/unit_tests/TestOpenMP_Sort1D.cpp index a9b2010ad0..4a5839f0c8 100644 --- a/lib/kokkos/algorithms/unit_tests/TestOpenMP_Sort1D.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestOpenMP_Sort1D.cpp @@ -59,6 +59,8 @@ TEST(openmp, SortUnsigned1D) { Impl::test_1D_sort(171); } +TEST(openmp, SortIssue1160) { Impl::test_issue_1160_sort(); } + } // namespace Test #else void KOKKOS_ALGORITHMS_UNITTESTS_TESTOPENMP_PREVENT_LINK_ERROR() {} diff --git a/lib/kokkos/algorithms/unit_tests/TestRandom.hpp b/lib/kokkos/algorithms/unit_tests/TestRandom.hpp index caba92c152..1f14875096 100644 --- a/lib/kokkos/algorithms/unit_tests/TestRandom.hpp +++ b/lib/kokkos/algorithms/unit_tests/TestRandom.hpp @@ -491,6 +491,34 @@ void test_random(unsigned int num_draws) { } } // namespace Impl +template +void test_random_xorshift64() { +#if defined(KOKKOS_ENABLE_SYCL) || defined(KOKKOS_ENABLE_CUDA) || \ + defined(KOKKOS_ENABLE_HIP) + const int num_draws = 132141141; +#else // SERIAL, HPX, OPENMP + const int num_draws = 10240000; +#endif + Impl::test_random>(num_draws); + Impl::test_random>>( + num_draws); +} + +template +void test_random_xorshift1024() { +#if defined(KOKKOS_ENABLE_SYCL) || defined(KOKKOS_ENABLE_CUDA) || \ + defined(KOKKOS_ENABLE_HIP) + const int num_draws = 52428813; +#else // SERIAL, HPX, OPENMP + const int num_draws = 10130144; +#endif + Impl::test_random>( + num_draws); + Impl::test_random>>( + num_draws); +} } // namespace Test #endif // KOKKOS_TEST_UNORDERED_MAP_HPP diff --git a/lib/kokkos/algorithms/unit_tests/TestRandomCommon.hpp b/lib/kokkos/algorithms/unit_tests/TestRandomCommon.hpp new file mode 100644 index 0000000000..c6d3b59ae1 --- /dev/null +++ b/lib/kokkos/algorithms/unit_tests/TestRandomCommon.hpp @@ -0,0 +1,60 @@ +/* +//@HEADER +// ************************************************************************ +// +// Kokkos v. 3.0 +// Copyright (2020) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact Christian R. Trott (crtrott@sandia.gov) +// +// ************************************************************************ +//@HEADER +*/ + +#ifndef KOKKOS_ALGORITHMS_UNITTESTS_TESTRANDOM_COMMON_HPP +#define KOKKOS_ALGORITHMS_UNITTESTS_TESTRANDOM_COMMON_HPP + +#include + +namespace Test { + +TEST(TEST_CATEGORY, Random_XorShift64) { + test_random_xorshift64(); +} +TEST(TEST_CATEGORY, Random_XorShift1024_0) { + test_random_xorshift1024(); +} +} // namespace Test + +#endif diff --git a/lib/kokkos/containers/unit_tests/TestHIP_Category.hpp b/lib/kokkos/algorithms/unit_tests/TestSortCommon.hpp similarity index 88% rename from lib/kokkos/containers/unit_tests/TestHIP_Category.hpp rename to lib/kokkos/algorithms/unit_tests/TestSortCommon.hpp index c2d60d1814..56657b6574 100644 --- a/lib/kokkos/containers/unit_tests/TestHIP_Category.hpp +++ b/lib/kokkos/algorithms/unit_tests/TestSortCommon.hpp @@ -42,10 +42,14 @@ //@HEADER */ -#ifndef KOKKOS_TEST_HIP_HPP -#define KOKKOS_TEST_HIP_HPP +#ifndef KOKKOS_ALGORITHMS_UNITTESTS_TESTSORT_COMMON_HPP +#define KOKKOS_ALGORITHMS_UNITTESTS_TESTSORT_COMMON_HPP -#define TEST_CATEGORY hip -#define TEST_EXECSPACE Kokkos::Experimental::HIP +#include +namespace Test { +TEST(TEST_CATEGORY, SortUnsigned) { + Impl::test_sort(171); +} +} // namespace Test #endif diff --git a/lib/kokkos/appveyor.yml b/lib/kokkos/appveyor.yml index c40bf066b7..e8763c0b66 100644 --- a/lib/kokkos/appveyor.yml +++ b/lib/kokkos/appveyor.yml @@ -3,8 +3,4 @@ image: clone_folder: c:\projects\source build_script: - cmd: >- - mkdir build && - cd build && - cmake c:\projects\source -DKokkos_ENABLE_TESTS=ON && - cmake --build . --target install && - ctest -C Debug -V + cmake c:\projects\source -DKokkos_ENABLE_TESTS=ON -DCMAKE_CXX_FLAGS="/W0 /EHsc /d1reportClassLayoutChanges" -DCTEST_ARGS="-C Debug -V --output-on-failure" -DBUILD_NAME=MSVC-2019 -DBUILD_TYPE=Debug -DSITE=AppVeyor -DTARGET=install -P cmake/KokkosCI.cmake diff --git a/lib/kokkos/bin/kokkos_launch_compiler b/lib/kokkos/bin/kokkos_launch_compiler index 1fbebf648f..d929d24f1d 100755 --- a/lib/kokkos/bin/kokkos_launch_compiler +++ b/lib/kokkos/bin/kokkos_launch_compiler @@ -13,6 +13,17 @@ # $1 are 'ar', 'cmake', etc. during the linking phase # +# emit a message about the underlying command executed +: ${DEBUG:=0} +: ${KOKKOS_DEBUG_LAUNCH_COMPILER:=${DEBUG}} + +debug-message() +{ + if [ "${KOKKOS_DEBUG_LAUNCH_COMPILER}" -ne 0 ]; then + echo -e "##### $(basename ${BASH_SOURCE[0]}) executing: \"$@\"... #####" + fi +} + # check the arguments for the KOKKOS_DEPENDENCE compiler definition KOKKOS_DEPENDENCE=0 for i in ${@} @@ -23,16 +34,30 @@ do fi done -# if C++ is not passed, someone is probably trying to invoke it directly +# if Kokkos compiler is not passed, someone is probably trying to invoke it directly if [ -z "${1}" ]; then - echo -e "\n${BASH_SOURCE[0]} was invoked without the C++ compiler as the first argument." + echo -e "\n${BASH_SOURCE[0]} was invoked without the Kokkos compiler as the first argument." echo "This script is not indended to be directly invoked by any mechanism other" - echo -e "than through a RULE_LAUNCH_COMPILE or RULE_LAUNCH_LINK property set in CMake\n" + echo -e "than through a RULE_LAUNCH_COMPILE or RULE_LAUNCH_LINK property set in CMake.\n" + exit 1 +fi + +# if Kokkos compiler is not passed, someone is probably trying to invoke it directly +if [ -z "${2}" ]; then + echo -e "\n${BASH_SOURCE[0]} was invoked without the C++ compiler as the second argument." + echo "This script is not indended to be directly invoked by any mechanism other" + echo -e "than through a RULE_LAUNCH_COMPILE or RULE_LAUNCH_LINK property set in CMake.\n" exit 1 fi # if there aren't two args, this isn't necessarily invalid, just a bit strange -if [ -z "${2}" ]; then exit 0; fi +if [ -z "${3}" ]; then exit 0; fi + +# store the Kokkos compiler +KOKKOS_COMPILER=${1} + +# remove the Kokkos compiler from the arguments +shift # store the expected C++ compiler CXX_COMPILER=${1} @@ -40,48 +65,57 @@ CXX_COMPILER=${1} # remove the expected C++ compiler from the arguments shift -# after the above shift, $1 is now the exe for the compile or link command, e.g. -# kokkos_launch_compiler g++ gcc -c file.c -o file.o +# NOTE: in below, ${KOKKOS_COMPILER} is usually nvcc_wrapper +# +# after the above shifts, $1 is now the exe for the compile or link command, e.g. +# kokkos_launch_compiler ${KOKKOS_COMPILER} g++ gcc -c file.c -o file.o # becomes: # kokkos_launch_compiler gcc -c file.c -o file.o -# Check to see if the executable is the C++ compiler and if it is not, then +# We check to see if the executable is the C++ compiler and if it is not, then # just execute the command. # # Summary: -# kokkos_launch_compiler g++ gcc -c file.c -o file.o +# kokkos_launch_compiler ${KOKKOS_COMPILER} g++ gcc -c file.c -o file.o # results in this command being executed: # gcc -c file.c -o file.o # and -# kokkos_launch_compiler g++ g++ -c file.cpp -o file.o +# kokkos_launch_compiler ${KOKKOS_COMPILER} g++ g++ -c file.cpp -o file.o # results in this command being executed: -# nvcc_wrapper -c file.cpp -o file.o +# ${KOKKOS_COMPILER} -c file.cpp -o file.o if [[ "${KOKKOS_DEPENDENCE}" -eq "0" || "${CXX_COMPILER}" != "${1}" ]]; then - # the command does not depend on Kokkos so just execute the command w/o re-directing to nvcc_wrapper + debug-message $@ + # the command does not depend on Kokkos so just execute the command w/o re-directing to ${KOKKOS_COMPILER} eval $@ else - # the executable is the C++ compiler, so we need to re-direct to nvcc_wrapper - - # find the nvcc_wrapper from the same build/install - NVCC_WRAPPER="$(dirname ${BASH_SOURCE[0]})/nvcc_wrapper" - - if [ -z "${NVCC_WRAPPER}" ]; then - echo -e "\nError: nvcc_wrapper not found in $(dirname ${BASH_SOURCE[0]}).\n" + # the executable is the C++ compiler, so we need to re-direct to ${KOKKOS_COMPILER} + if [ ! -f "${KOKKOS_COMPILER}" ]; then + echo -e "\nError: the compiler redirect for Kokkos was not found at ${KOKKOS_COMPILER}\n" exit 1 fi - # set default nvcc wrapper compiler if not specified - : ${NVCC_WRAPPER_DEFAULT_COMPILER:=${CXX_COMPILER}} - export NVCC_WRAPPER_DEFAULT_COMPILER + # find the nvcc_wrapper from the same build/install + NVCC_WRAPPER="$(dirname ${BASH_SOURCE[0]})/nvcc_wrapper" + if [ "${KOKKOS_COMPILER}" = "${NVCC_WRAPPER}" ]; then + # this should only be valid in the install tree -- it will be set to CMAKE_CXX_COMPILER used using Kokkos installation + if [ -z $(echo "@NVCC_WRAPPER_DEFAULT_COMPILER@" | grep 'NVCC_WRAPPER_DEFAULT_COMPILER') ]; then + : ${NVCC_WRAPPER_DEFAULT_COMPILER:="@NVCC_WRAPPER_DEFAULT_COMPILER@"} + fi - # calling itself will cause an infinitely long build - if [ "${NVCC_WRAPPER}" = "${NVCC_WRAPPER_DEFAULT_COMPILER}" ]; then - echo -e "\nError: NVCC_WRAPPER == NVCC_WRAPPER_DEFAULT_COMPILER. Terminating to avoid infinite loop!\n" - exit 1 + # set default nvcc wrapper compiler if not specified + : ${NVCC_WRAPPER_DEFAULT_COMPILER:=${CXX_COMPILER}} + export NVCC_WRAPPER_DEFAULT_COMPILER + + # nvcc_wrapper calling itself will cause an infinitely long build + if [ "${NVCC_WRAPPER}" = "${NVCC_WRAPPER_DEFAULT_COMPILER}" ]; then + echo -e "\nError: NVCC_WRAPPER == NVCC_WRAPPER_DEFAULT_COMPILER. Terminating to avoid infinite loop!\n" + exit 1 + fi fi # discard the compiler from the command shift - # execute nvcc_wrapper - ${NVCC_WRAPPER} $@ + debug-message ${KOKKOS_COMPILER} $@ + # execute ${KOKKOS_COMPILER} (again, usually nvcc_wrapper) + ${KOKKOS_COMPILER} $@ fi diff --git a/lib/kokkos/bin/nvcc_wrapper b/lib/kokkos/bin/nvcc_wrapper index 4ecf4c66d5..5556e888e3 100755 --- a/lib/kokkos/bin/nvcc_wrapper +++ b/lib/kokkos/bin/nvcc_wrapper @@ -191,11 +191,11 @@ do shift ;; #Handle known nvcc args - --dryrun|--verbose|--keep|--keep-dir*|-G|--relocatable-device-code*|-lineinfo|-expt-extended-lambda|-expt-relaxed-constexpr|--resource-usage|-Xptxas*|--fmad*|--Wext-lambda-captures-this|-Wext-lambda-captures-this) + --dryrun|--verbose|--keep|--keep-dir*|-G|--relocatable-device-code*|-lineinfo|-expt-extended-lambda|-expt-relaxed-constexpr|--resource-usage|-Xptxas*|--fmad*|--use_fast_math|--Wext-lambda-captures-this|-Wext-lambda-captures-this) cuda_args="$cuda_args $1" ;; #Handle more known nvcc args - --expt-extended-lambda|--expt-relaxed-constexpr) + --expt-extended-lambda|--expt-relaxed-constexpr|--Wno-deprecated-gpu-targets|-Wno-deprecated-gpu-targets) cuda_args="$cuda_args $1" ;; #Handle known nvcc args that have an argument diff --git a/lib/kokkos/cmake/CTestConfig.cmake.in b/lib/kokkos/cmake/CTestConfig.cmake.in new file mode 100644 index 0000000000..1f82c0d64d --- /dev/null +++ b/lib/kokkos/cmake/CTestConfig.cmake.in @@ -0,0 +1,91 @@ +#----------------------------------------------------------------------------------------# +# +# CTestConfig.cmake template for Kokkos +# +#----------------------------------------------------------------------------------------# + +# +# dash-board related +# +set(CTEST_PROJECT_NAME "Kokkos") +set(CTEST_NIGHTLY_START_TIME "01:00:00 UTC") +set(CTEST_DROP_METHOD "https") +set(CTEST_DROP_SITE "cdash.nersc.gov") +set(CTEST_DROP_LOCATION "/submit.php?project=${CTEST_PROJECT_NAME}") +set(CTEST_CDASH_VERSION "1.6") +set(CTEST_CDASH_QUERY_VERSION TRUE) +set(CTEST_SUBMIT_RETRY_COUNT "1") +set(CTEST_SUBMIT_RETRY_DELAY "30") + +# +# configure/build related +# +set(CTEST_BUILD_NAME "@BUILD_NAME@") +set(CTEST_MODEL "@MODEL@") +set(CTEST_SITE "@SITE@") +set(CTEST_CONFIGURATION_TYPE "@BUILD_TYPE@") +set(CTEST_SOURCE_DIRECTORY "@SOURCE_REALDIR@") +set(CTEST_BINARY_DIRECTORY "@BINARY_REALDIR@") + +# +# configure/build related +# +set(CTEST_UPDATE_TYPE "git") +set(CTEST_UPDATE_VERSION_ONLY ON) +# set(CTEST_GENERATOR "") +# set(CTEST_GENERATOR_PLATFORM "") + +# +# testing related +# +set(CTEST_TIMEOUT "7200") +set(CTEST_TEST_TIMEOUT "7200") +set(CTEST_CUSTOM_MAXIMUM_NUMBER_OF_ERRORS "100") +set(CTEST_CUSTOM_MAXIMUM_NUMBER_OF_WARNINGS "100") +set(CTEST_CUSTOM_MAXIMUM_PASSED_TEST_OUTPUT_SIZE "1048576") + +# +# coverage related +# +set(CTEST_CUSTOM_COVERAGE_EXCLUDE ".*tpls/.*;/usr/.*;.*unit_test/.*;.*unit_tests/.*;.*perf_test/.*") + +# +# commands +# +if(NOT "@CHECKOUT_COMMAND@" STREQUAL "") + set(CTEST_CHECKOUT_COMMAND "@CHECKOUT_COMMAND@") +endif() +set(CTEST_UPDATE_COMMAND "@GIT_EXECUTABLE@") +set(CTEST_CONFIGURE_COMMAND "@CMAKE_COMMAND@ -DCMAKE_BUILD_TYPE=@BUILD_TYPE@ -DKokkos_ENABLE_TESTS=ON @CONFIG_ARGS@ @SOURCE_REALDIR@") +set(CTEST_BUILD_COMMAND "@CMAKE_COMMAND@ --build @BINARY_REALDIR@ --target @TARGET@") +if(NOT WIN32) + set(CTEST_BUILD_COMMAND "${CTEST_BUILD_COMMAND} -- -j@BUILD_JOBS@") +endif() +set(CTEST_COVERAGE_COMMAND "gcov") +set(CTEST_MEMORYCHECK_COMMAND "valgrind") +set(CTEST_GIT_COMMAND "@GIT_EXECUTABLE@") + +# +# various configs +# +set(APPEND_VALUE @APPEND@) +if(APPEND_VALUE) + set(APPEND_CTEST APPEND) +endif() + +macro(SET_TEST_PROP VAR) + if(NOT "${ARGS}" STREQUAL "") + set(${VAR}_CTEST ${VAR} ${ARGN}) + endif() +endmacro() + +set_test_prop(START @START@) +set_test_prop(END @END@) +set_test_prop(STRIDE @STRIDE@) +set_test_prop(INCLUDE @INCLUDE@) +set_test_prop(EXCLUDE @EXCLUDE@) +set_test_prop(INCLUDE_LABEL @INCLUDE_LABEL@) +set_test_prop(EXCLUDE_LABEL @EXCLUDE_LABEL@) +set_test_prop(PARALLEL_LEVEL @PARALLEL_LEVEL@) +set_test_prop(STOP_TIME @STOP_TIME@) +set_test_prop(COVERAGE_LABELS @LABELS@) diff --git a/lib/kokkos/cmake/KokkosCI.cmake b/lib/kokkos/cmake/KokkosCI.cmake new file mode 100644 index 0000000000..e8c9af37ad --- /dev/null +++ b/lib/kokkos/cmake/KokkosCI.cmake @@ -0,0 +1,350 @@ +cmake_minimum_required(VERSION 3.16 FATAL_ERROR) + +message(STATUS "") + +get_cmake_property(_cached_vars CACHE_VARIABLES) +set(KOKKOS_CMAKE_ARGS) +set(EXCLUDED_VARIABLES "CMAKE_COMMAND" "CMAKE_CPACK_COMMAND" "CMAKE_CTEST_COMMAND" "CMAKE_ROOT" + "CTEST_ARGS" "BUILD_NAME" "CMAKE_CXX_FLAGS" "CMAKE_BUILD_TYPE") +list(SORT _cached_vars) +foreach(_var ${_cached_vars}) + if(NOT "${_var}" IN_LIST EXCLUDED_VARIABLES) + list(APPEND KOKKOS_CMAKE_ARGS ${_var}) + if("${_var}" STREQUAL "CMAKE_BUILD_TYPE") + set(BUILD_TYPE "${CMAKE_BUILD_TYPE}") + endif() + endif() +endforeach() + + +#----------------------------------------------------------------------------------------# +# +# Macros and variables +# +#----------------------------------------------------------------------------------------# + +macro(CHECK_REQUIRED VAR) + if(NOT DEFINED ${VAR}) + message(FATAL_ERROR "Error! Variable '${VAR}' must be defined") + endif() +endmacro() + +# require the build name variable +CHECK_REQUIRED(BUILD_NAME) + +# uses all args +macro(SET_DEFAULT VAR) + if(NOT DEFINED ${VAR}) + set(${VAR} ${ARGN}) + endif() + # remove these ctest configuration variables from the defines + # passed to the Kokkos configuration + if("${VAR}" IN_LIST KOKKOS_CMAKE_ARGS) + list(REMOVE_ITEM KOKKOS_CMAKE_ARGS "${VAR}") + endif() +endmacro() + +# uses first arg -- useful for selecting via priority from multiple +# potentially defined variables, e.g.: +# +# set_default_arg1(BUILD_NAME ${TRAVIS_BUILD_NAME} ${BUILD_NAME}) +# +macro(SET_DEFAULT_ARG1 VAR) + if(NOT DEFINED ${VAR}) + foreach(_ARG ${ARGN}) + if(NOT "${_ARG}" STREQUAL "") + set(${VAR} ${_ARG}) + break() + endif() + endforeach() + endif() + # remove these ctest configuration variables from the defines + # passed to the Kokkos configuration + if("${VAR}" IN_LIST KOKKOS_CMAKE_ARGS) + list(REMOVE_ITEM KOKKOS_CMAKE_ARGS "${VAR}") + endif() +endmacro() + +# determine the default working directory +if(NOT "$ENV{WORKSPACE}" STREQUAL "") + set(WORKING_DIR "$ENV{WORKSPACE}") +else() + get_filename_component(WORKING_DIR ${CMAKE_CURRENT_LIST_DIR} DIRECTORY) +endif() + +# determine the hostname +execute_process(COMMAND hostname + OUTPUT_VARIABLE HOSTNAME + OUTPUT_STRIP_TRAILING_WHITESPACE) + +SET_DEFAULT(HOSTNAME "$ENV{HOSTNAME}") + +# get the number of processors +include(ProcessorCount) +ProcessorCount(NUM_PROCESSORS) + +# find git +find_package(Git QUIET) +if(NOT GIT_EXECUTABLE) + unset(GIT_EXECUTABLE CACHE) + unset(GIT_EXECUTABLE) +endif() + +function(EXECUTE_GIT_COMMAND VAR) + set(${VAR} "" PARENT_SCOPE) + execute_process(COMMAND ${GIT_EXECUTABLE} ${ARGN} + OUTPUT_VARIABLE VAL + RESULT_VARIABLE RET + OUTPUT_STRIP_TRAILING_WHITESPACE + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} + ERROR_QUIET) + string(REPLACE ";" " " _CMD "${GIT_EXECUTABLE} ${ARGN}") + set(LAST_GIT_COMMAND "${_CMD}" PARENT_SCOPE) + if(RET EQUAL 0) + set(${VAR} "${VAL}" PARENT_SCOPE) + endif() +endfunction() + +# just gets the git branch name if available +function(GET_GIT_BRANCH_NAME VAR) + execute_git_command(GIT_BRANCH branch --show-current) + set(_INVALID "%D" "HEAD") + if(NOT GIT_BRANCH OR "${GIT_BRANCH}" IN_LIST _INVALID) + execute_git_command(GIT_BRANCH show -s --format=%D) + if(NOT GIT_BRANCH OR "${GIT_BRANCH}" IN_LIST _INVALID) + execute_git_command(GIT_BRANCH --describe all) + endif() + endif() + # + if(GIT_BRANCH) + string(REPLACE " " ";" _DESC "${GIT_BRANCH}") + # just set it to last one via loop instead of wonky cmake index manip + foreach(_ITR ${_DESC}) + set(GIT_BRANCH "${_ITR}") + endforeach() + set(${VAR} "${GIT_BRANCH}" PARENT_SCOPE) + message(STATUS "GIT BRANCH via '${LAST_GIT_COMMAND}': ${GIT_BRANCH}") + endif() +endfunction() + +# just gets the git branch name if available +function(GET_GIT_AUTHOR_NAME VAR) + execute_git_command(GIT_AUTHOR show -s --format=%an) + if(GIT_AUTHOR) + string(LENGTH "${GIT_AUTHOR}" STRLEN) + # if the build name gets too long, this can cause submission errors + if(STRLEN GREATER 24) + # remove middle initial + string(REGEX REPLACE " [A-Z]\. " " " GIT_AUTHOR "${GIT_AUTHOR}") + # get first and sur name + string(REGEX REPLACE "([A-Za-z]+) ([A-Za-z]+)" "\\1" F_NAME "${GIT_AUTHOR}") + string(REGEX REPLACE "([A-Za-z]+) ([A-Za-z]+)" "\\2" S_NAME "${GIT_AUTHOR}") + if(S_NAME) + set(GIT_AUTHOR "${S_NAME}") + elseif(F_NAME) + set(GIT_AUTHOR "${F_NAME}") + endif() + endif() + # remove any spaces, quotes, periods, etc. + string(REGEX REPLACE "[ ',;_\.\"]+" "" GIT_AUTHOR "${GIT_AUTHOR}") + set(${VAR} "${GIT_AUTHOR}" PARENT_SCOPE) + message(STATUS "GIT AUTHOR via '${LAST_GIT_COMMAND}': ${GIT_AUTHOR}") + endif() +endfunction() + +# get the name of the branch +GET_GIT_BRANCH_NAME(GIT_BRANCH) +# get the name of the author +GET_GIT_AUTHOR_NAME(GIT_AUTHOR) +# author, prefer git method for consistency +SET_DEFAULT_ARG1(AUTHOR ${GIT_AUTHOR} $ENV{GIT_AUTHOR} $ENV{AUTHOR}) +# SLUG == owner_name/repo_name +SET_DEFAULT_ARG1(SLUG $ENV{TRAVIS_PULL_REQUEST_SLUG} $ENV{TRAVIS_REPO_SLUG} $ENV{APPVEYOR_REPO_NAME} $ENV{PULL_REQUEST_SLUG} $ENV{REPO_SLUG}) +# branch name +SET_DEFAULT_ARG1(BRANCH $ENV{TRAVIS_PULL_REQUEST_BRANCH} $ENV{TRAVIS_BRANCH} $ENV{APPVEYOR_PULL_REQUEST_HEAD_REPO_BRANCH} $ENV{APPVEYOR_REPO_BRANCH} $ENV{GIT_BRANCH} $ENV{BRANCH_NAME} $ENV{BRANCH} ${GIT_BRANCH}) +# pull request number +SET_DEFAULT_ARG1(PULL_REQUEST_NUM $ENV{TRAVIS_PULL_REQUEST} $ENV{CHANGE_ID} $ENV{APPVEYOR_PULL_REQUEST_NUMBER} $ENV{PULL_REQUEST_NUM}) +# get the event type, e.g. push, pull_request, api, cron, etc. +SET_DEFAULT_ARG1(EVENT_TYPE $ENV{TRAVIS_EVENT_TYPE} ${EVENT_TYPE}) + +if("${BRANCH}" STREQUAL "") + message(STATUS "Checked: environment variables for Travis, Appveyor, Jenkins (git plugin), BRANCH_NAME, BRANCH and 'git branch --show-current'") + message(FATAL_ERROR "Error! Git branch could not be determined. Please provide -DBRANCH=") +endif() + +#----------------------------------------------------------------------------------------# +# +# Set default values if not provided on command-line +# +#----------------------------------------------------------------------------------------# + +SET_DEFAULT(SOURCE_DIR "${WORKING_DIR}") # source directory +SET_DEFAULT(BINARY_DIR "${WORKING_DIR}/build") # build directory +SET_DEFAULT(BUILD_TYPE "${CMAKE_BUILD_TYPE}") # Release, Debug, etc. +SET_DEFAULT(MODEL "Continuous") # Continuous, Nightly, or Experimental +SET_DEFAULT(JOBS 1) # number of parallel ctests +SET_DEFAULT(CTEST_COMMAND "${CMAKE_CTEST_COMMAND}") # just in case +SET_DEFAULT(CTEST_ARGS "-V --output-on-failure") # extra arguments when ctest is called +SET_DEFAULT(GIT_EXECUTABLE "git") # ctest_update +SET_DEFAULT(TARGET "all") # build target +SET_DEFAULT_ARG1(SITE "$ENV{SITE}" + "${HOSTNAME}") # update site +SET_DEFAULT_ARG1(BUILD_JOBS "$ENV{BUILD_JOBS}" + "${NUM_PROCESSORS}") # number of parallel compile jobs +# +# The variable below correspond to ctest arguments, i.e. START,END,STRIDE are +# '-I START,END,STRIDE' +# +SET_DEFAULT(START "") +SET_DEFAULT(END "") +SET_DEFAULT(STRIDE "") +SET_DEFAULT(INCLUDE "") +SET_DEFAULT(EXCLUDE "") +SET_DEFAULT(INCLUDE_LABEL "") +SET_DEFAULT(EXCLUDE_LABEL "") +SET_DEFAULT(PARALLEL_LEVEL "") +SET_DEFAULT(STOP_TIME "") +SET_DEFAULT(LABELS "") +SET_DEFAULT(NOTES "") + +# default static build tag for Nightly +set(BUILD_TAG "${BRANCH}") + +if(NOT BUILD_TYPE) + # default for kokkos if not specified + set(BUILD_TYPE "RelWithDebInfo") +endif() + +# generate dynamic name if continuous or experimental model +if(NOT "${MODEL}" STREQUAL "Nightly") + if(EVENT_TYPE AND PULL_REQUEST_NUM) + # e.g. pull_request/123 + if(AUTHOR) + set(BUILD_TAG "${AUTHOR}/${EVENT_TYPE}/${PULL_REQUEST_NUM}") + else() + set(BUILD_TAG "${EVENT_TYPE}/${PULL_REQUEST_NUM}") + endif() + elseif(SLUG) + # e.g. owner_name/repo_name + set(BUILD_TAG "${SLUG}") + elseif(AUTHOR) + set(BUILD_TAG "${AUTHOR}/${BRANCH}") + endif() + if(EVENT_TYPE AND NOT PULL_REQUEST_NUM) + set(BUILD_TAG "${BUILD_TAG}-${EVENT_TYPE}") + endif() +endif() + +# unnecessary +string(REPLACE "/remotes/" "/" BUILD_TAG "${BUILD_TAG}") +string(REPLACE "/origin/" "/" BUILD_TAG "${BUILD_TAG}") + +message(STATUS "BUILD_TAG: ${BUILD_TAG}") + +set(BUILD_NAME "[${BUILD_TAG}] [${BUILD_NAME}-${BUILD_TYPE}]") + +# colons in build name create extra (empty) entries in CDash +string(REPLACE ":" "-" BUILD_NAME "${BUILD_NAME}") +# unnecessary info +string(REPLACE "/merge]" "]" BUILD_NAME "${BUILD_NAME}") +# consistency +string(REPLACE "/pr/" "/pull/" BUILD_NAME "${BUILD_NAME}") +string(REPLACE "pull_request/" "pull/" BUILD_NAME "${BUILD_NAME}") +# miscellaneous from missing fields +string(REPLACE "--" "-" BUILD_NAME "${BUILD_NAME}") +string(REPLACE "-]" "]" BUILD_NAME "${BUILD_NAME}") + +# check binary directory +if(EXISTS ${BINARY_DIR}) + if(NOT IS_DIRECTORY "${BINARY_DIR}") + message(FATAL_ERROR "Error! '${BINARY_DIR}' already exists and is not a directory!") + endif() + file(GLOB BINARY_DIR_FILES "${BINARY_DIR}/*") + if(NOT "${BINARY_DIR_FILES}" STREQUAL "") + message(FATAL_ERROR "Error! '${BINARY_DIR}' already exists and is not empty!") + endif() +endif() + +get_filename_component(SOURCE_REALDIR ${SOURCE_DIR} REALPATH) +get_filename_component(BINARY_REALDIR ${BINARY_DIR} REALPATH) + +#----------------------------------------------------------------------------------------# +# +# Generate the CTestConfig.cmake +# +#----------------------------------------------------------------------------------------# + +set(CONFIG_ARGS) +foreach(_ARG ${KOKKOS_CMAKE_ARGS}) + if(NOT "${${_ARG}}" STREQUAL "") + get_property(_ARG_TYPE CACHE ${_ARG} PROPERTY TYPE) + if("${_ARG_TYPE}" STREQUAL "UNINITIALIZED") + if("${${_ARG}}" STREQUAL "ON" OR "${${_ARG}}" STREQUAL "OFF") + set(_ARG_TYPE "BOOL") + elseif(EXISTS "${${_ARG}}" AND NOT IS_DIRECTORY "${${_ARG}}") + set(_ARG_TYPE "FILEPATH") + elseif(EXISTS "${${_ARG}}" AND IS_DIRECTORY "${${_ARG}}") + set(_ARG_TYPE "PATH") + elseif(NOT "${${_ARG}}" STREQUAL "") + set(_ARG_TYPE "STRING") + endif() + endif() + set(CONFIG_ARGS "${CONFIG_ARGS}set(${_ARG} \"${${_ARG}}\" CACHE ${_ARG_TYPE} \"\")\n") + endif() +endforeach() + +file(WRITE ${BINARY_REALDIR}/initial-cache.cmake +" +set(CMAKE_CXX_FLAGS \"${CMAKE_CXX_FLAGS}\" CACHE STRING \"\") +${CONFIG_ARGS} +") + +file(READ ${BINARY_REALDIR}/initial-cache.cmake _CACHE_INFO) +message(STATUS "Initial cache:\n${_CACHE_INFO}") + +# initialize the cache +set(CONFIG_ARGS "-C ${BINARY_REALDIR}/initial-cache.cmake") + + +# generate the CTestConfig.cmake +configure_file( + ${CMAKE_CURRENT_LIST_DIR}/CTestConfig.cmake.in + ${BINARY_REALDIR}/CTestConfig.cmake + @ONLY) + +# copy/generate the dashboard script +configure_file( + ${CMAKE_CURRENT_LIST_DIR}/KokkosCTest.cmake.in + ${BINARY_REALDIR}/KokkosCTest.cmake + @ONLY) + +# custom CTest settings go in ${BINARY_DIR}/CTestCustom.cmake +execute_process( + COMMAND ${CMAKE_COMMAND} -E touch CTestCustom.cmake + WORKING_DIRECTORY ${BINARY_REALDIR} + ) + +#----------------------------------------------------------------------------------------# +# +# Execute CTest +# +#----------------------------------------------------------------------------------------# + +message(STATUS "") +message(STATUS "BUILD_NAME: ${BUILD_NAME}") +message(STATUS "Executing '${CTEST_COMMAND} -S KokkosCTest.cmake ${CTEST_ARGS}'...") +message(STATUS "") + +# e.g. -DCTEST_ARGS="--output-on-failure -VV" should really be -DCTEST_ARGS="--output-on-failure;-VV" +string(REPLACE " " ";" CTEST_ARGS "${CTEST_ARGS}") + +execute_process( + COMMAND ${CTEST_COMMAND} -S KokkosCTest.cmake ${CTEST_ARGS} + RESULT_VARIABLE RET + WORKING_DIRECTORY ${BINARY_REALDIR} + ) + +# ensure that any non-zero result variable gets propagated +if(NOT RET EQUAL 0) + message(FATAL_ERROR "CTest return non-zero exit code: ${RET}") +endif() diff --git a/lib/kokkos/cmake/KokkosCTest.cmake.in b/lib/kokkos/cmake/KokkosCTest.cmake.in new file mode 100644 index 0000000000..b6917f3cc1 --- /dev/null +++ b/lib/kokkos/cmake/KokkosCTest.cmake.in @@ -0,0 +1,261 @@ +cmake_minimum_required(VERSION 3.16 FATAL_ERROR) + +if(EXISTS "${CMAKE_CURRENT_LIST_DIR}/CTestConfig.cmake") + include("${CMAKE_CURRENT_LIST_DIR}/CTestConfig.cmake") +endif() + +include(ProcessorCount) +ProcessorCount(CTEST_PROCESSOR_COUNT) + +cmake_policy(SET CMP0009 NEW) +cmake_policy(SET CMP0011 NEW) + +# ---------------------------------------------------------------------------- # +# -- Commands +# ---------------------------------------------------------------------------- # +find_program(CTEST_CMAKE_COMMAND NAMES cmake) +find_program(CTEST_UNAME_COMMAND NAMES uname) + +find_program(CTEST_BZR_COMMAND NAMES bzr) +find_program(CTEST_CVS_COMMAND NAMES cvs) +find_program(CTEST_GIT_COMMAND NAMES git) +find_program(CTEST_HG_COMMAND NAMES hg) +find_program(CTEST_P4_COMMAND NAMES p4) +find_program(CTEST_SVN_COMMAND NAMES svn) + +find_program(VALGRIND_COMMAND NAMES valgrind) +find_program(GCOV_COMMAND NAMES gcov) +find_program(LCOV_COMMAND NAMES llvm-cov) +find_program(MEMORYCHECK_COMMAND NAMES valgrind ) + +set(MEMORYCHECK_TYPE Valgrind) +# set(MEMORYCHECK_TYPE Purify) +# set(MEMORYCHECK_TYPE BoundsChecker) +# set(MEMORYCHECK_TYPE ThreadSanitizer) +# set(MEMORYCHECK_TYPE AddressSanitizer) +# set(MEMORYCHECK_TYPE LeakSanitizer) +# set(MEMORYCHECK_TYPE MemorySanitizer) +# set(MEMORYCHECK_TYPE UndefinedBehaviorSanitizer) +set(MEMORYCHECK_COMMAND_OPTIONS "--trace-children=yes --leak-check=full") + +# ---------------------------------------------------------------------------- # +# -- Settings +# ---------------------------------------------------------------------------- # +## -- Process timeout in seconds +set(CTEST_TIMEOUT "7200") +## -- Set output to English +set(ENV{LC_MESSAGES} "en_EN" ) + + +# ---------------------------------------------------------------------------- # +# -- Copy ctest configuration file +# ---------------------------------------------------------------------------- # +macro(COPY_CTEST_CONFIG_FILES) + + foreach(_FILE CTestConfig.cmake CTestCustom.cmake) + + # if current directory is not binary or source directory + if(NOT "${CMAKE_CURRENT_LIST_DIR}" STREQUAL "${CTEST_BINARY_DIRECTORY}" AND + NOT "${CTEST_SOURCE_DIRECTORY}" STREQUAL "${CTEST_BINARY_DIRECTORY}") + + # if file exists in current directory + if(EXISTS ${CMAKE_CURRENT_LIST_DIR}/${_FILE}) + configure_file(${CMAKE_CURRENT_LIST_DIR}/${_FILE} + ${CTEST_BINARY_DIRECTORY}/${_FILE} COPYONLY) + endif() + + # if source and binary differ + elseif(NOT "${CTEST_SOURCE_DIRECTORY}" STREQUAL "${CTEST_BINARY_DIRECTORY}") + + # if file exists in source directory but not in binary directory + if(EXISTS ${CTEST_SOURCE_DIRECTORY}/${_FILE} AND + NOT EXISTS ${CTEST_BINARY_DIRECTORY}/${_FILE}) + configure_file(${CTEST_SOURCE_DIRECTORY}/${_FILE} + ${CTEST_BINARY_DIRECTORY}/${_FILE} COPYONLY) + endif() + + endif() + endforeach() + +endmacro() + +ctest_read_custom_files("${CMAKE_CURRENT_LIST_DIR}") + +message(STATUS "CTEST_MODEL: ${CTEST_MODEL}") + +#-------------------------------------------------------------------------# +# Start +# +message(STATUS "") +message(STATUS "[${CTEST_BUILD_NAME}] Running START_CTEST stage...") +message(STATUS "") + +ctest_start(${CTEST_MODEL} TRACK ${CTEST_MODEL} ${APPEND_CTEST} + ${CTEST_SOURCE_DIRECTORY} ${CTEST_BINARY_DIRECTORY}) + + +#-------------------------------------------------------------------------# +# Config +# +copy_ctest_config_files() +ctest_read_custom_files("${CTEST_BINARY_DIRECTORY}") + + +#-------------------------------------------------------------------------# +# Update +# +message(STATUS "") +message(STATUS "[${CTEST_BUILD_NAME}] Running CTEST_UPDATE stage...") +message(STATUS "") + +ctest_update(SOURCE "${CTEST_SOURCE_DIRECTORY}" + RETURN_VALUE up_ret) + + +#-------------------------------------------------------------------------# +# Configure +# +message(STATUS "") +message(STATUS "[${CTEST_BUILD_NAME}] Running CTEST_CONFIGURE stage...") +message(STATUS "") + +ctest_configure(BUILD "${CTEST_BINARY_DIRECTORY}" + SOURCE ${CTEST_SOURCE_DIRECTORY} + ${APPEND_CTEST} + OPTIONS "${CTEST_CONFIGURE_OPTIONS}" + RETURN_VALUE config_ret) + + +#-------------------------------------------------------------------------# +# Echo configure log bc Damien wants to delay merging this PR for eternity +# +file(GLOB _configure_log "${CTEST_BINARY_DIRECTORY}/Testing/Temporary/LastConfigure*.log") +# should only have one but loop just for safety +foreach(_LOG ${_configure_log}) + file(READ ${_LOG} _LOG_MESSAGE) + message(STATUS "Configure Log: ${_LOG}") + message(STATUS "\n${_LOG_MESSAGE}\n") +endforeach() + + +#-------------------------------------------------------------------------# +# Build +# +message(STATUS "") +message(STATUS "[${CTEST_BUILD_NAME}] Running CTEST_BUILD stage...") +message(STATUS "") + +ctest_build(BUILD "${CTEST_BINARY_DIRECTORY}" + ${APPEND_CTEST} + RETURN_VALUE build_ret) + + +#-------------------------------------------------------------------------# +# Echo build log bc Damien wants to delay merging this PR for eternity +# +file(GLOB _build_log "${CTEST_BINARY_DIRECTORY}/Testing/Temporary/LastBuild*.log") +# should only have one but loop just for safety +foreach(_LOG ${_build_log}) + file(READ ${_LOG} _LOG_MESSAGE) + message(STATUS "Build Log: ${_LOG}") + message(STATUS "\n${_LOG_MESSAGE}\n") +endforeach() + + +#-------------------------------------------------------------------------# +# Test +# +message(STATUS "") +message(STATUS "[${CTEST_BUILD_NAME}] Running CTEST_TEST stage...") +message(STATUS "") + +ctest_test(RETURN_VALUE test_ret + ${APPEND_CTEST} + ${START_CTEST} + ${END_CTEST} + ${STRIDE_CTEST} + ${INCLUDE_CTEST} + ${EXCLUDE_CTEST} + ${INCLUDE_LABEL_CTEST} + ${EXCLUDE_LABEL_CTEST} + ${PARALLEL_LEVEL_CTEST} + ${STOP_TIME_CTEST} + SCHEDULE_RANDOM OFF) + + +#-------------------------------------------------------------------------# +# Coverage +# +message(STATUS "") +message(STATUS "[${CTEST_BUILD_NAME}] Running CTEST_COVERAGE stage...") +message(STATUS "") + +execute_process(COMMAND ${CTEST_COVERAGE_COMMAND} ${CTEST_COVERAGE_EXTRA_FLAGS} + WORKING_DIRECTORY ${CTEST_BINARY_DIRECTORY} + ERROR_QUIET) + +ctest_coverage(${APPEND_CTEST} + ${CTEST_COVERAGE_LABELS} + RETURN_VALUE cov_ret) + + +#-------------------------------------------------------------------------# +# MemCheck +# +message(STATUS "") +message(STATUS "[${CTEST_BUILD_NAME}] Running CTEST_MEMCHECK stage...") +message(STATUS "") + +ctest_memcheck(RETURN_VALUE mem_ret + ${APPEND_CTEST} + ${START_CTEST} + ${END_CTEST} + ${STRIDE_CTEST} + ${INCLUDE_CTEST} + ${EXCLUDE_CTEST} + ${INCLUDE_LABEL_CTEST} + ${EXCLUDE_LABEL_CTEST} + ${PARALLEL_LEVEL_CTEST}) + + +#-------------------------------------------------------------------------# +# Submit +# +message(STATUS "") +message(STATUS "[${CTEST_BUILD_NAME}] Running CTEST_SUBMIT stage...") +message(STATUS "") + +file(GLOB_RECURSE NOTE_FILES "${CTEST_BINARY_DIRECTORY}/*CTestNotes.cmake") +foreach(_FILE ${NOTE_FILES}) + message(STATUS "Including CTest notes files: \"${_FILE}\"...") + include("${_FILE}") +endforeach() + +# capture submit error so it doesn't fail because of a submission error +ctest_submit(RETURN_VALUE submit_ret + RETRY_COUNT 2 + RETRY_DELAY 10 + CAPTURE_CMAKE_ERROR submit_err) + +#-------------------------------------------------------------------------# +# Submit +# +message(STATUS "") +message(STATUS "[${CTEST_BUILD_NAME}] Finished ${CTEST_MODEL} Stages (${STAGES})") +message(STATUS "") + + +#-------------------------------------------------------------------------# +# Non-zero exit codes for important errors +# +if(NOT config_ret EQUAL 0) + message(FATAL_ERROR "Error during configuration! Exit code: ${config_ret}") +endif() + +if(NOT build_ret EQUAL 0) + message(FATAL_ERROR "Error during build! Exit code: ${build_ret}") +endif() + +if(NOT test_ret EQUAL 0) + message(FATAL_ERROR "Error during testing! Exit code: ${test_ret}") +endif() diff --git a/lib/kokkos/cmake/KokkosConfig.cmake.in b/lib/kokkos/cmake/KokkosConfig.cmake.in index 9fbd22ee5c..44a8fcd9c3 100644 --- a/lib/kokkos/cmake/KokkosConfig.cmake.in +++ b/lib/kokkos/cmake/KokkosConfig.cmake.in @@ -19,17 +19,44 @@ INCLUDE("${Kokkos_CMAKE_DIR}/KokkosTargets.cmake") INCLUDE("${Kokkos_CMAKE_DIR}/KokkosConfigCommon.cmake") UNSET(Kokkos_CMAKE_DIR) -# if CUDA was enabled and separable compilation was specified, e.g. -# find_package(Kokkos COMPONENTS separable_compilation) -# then we set the RULE_LAUNCH_COMPILE and RULE_LAUNCH_LINK -IF(@Kokkos_ENABLE_CUDA@ AND NOT "separable_compilation" IN_LIST Kokkos_FIND_COMPONENTS) +# check for conflicts +IF("launch_compiler" IN_LIST Kokkos_FIND_COMPONENTS AND + "separable_compilation" IN_LIST Kokkos_FIND_COMPONENTS) + MESSAGE(STATUS "'launch_compiler' implies global redirection of targets depending on Kokkos to appropriate compiler.") + MESSAGE(STATUS "'separable_compilation' implies explicitly defining where redirection occurs via 'kokkos_compilation(PROJECT|TARGET|SOURCE|DIRECTORY ...)'") + MESSAGE(FATAL_ERROR "Conflicting COMPONENTS: 'launch_compiler' and 'separable_compilation'") +ENDIF() + +IF("launch_compiler" IN_LIST Kokkos_FIND_COMPONENTS) + # + # if find_package(Kokkos COMPONENTS launch_compiler) then rely on the + # RULE_LAUNCH_COMPILE and RULE_LAUNCH_LINK to always redirect to the + # appropriate compiler for Kokkos + # + + MESSAGE(STATUS "kokkos_launch_compiler is enabled globally. C++ compiler commands with -DKOKKOS_DEPENDENCE will be redirected to the appropriate compiler for Kokkos") + kokkos_compilation( + GLOBAL + CHECK_CUDA_COMPILES) + +ELSEIF(@Kokkos_ENABLE_CUDA@ AND NOT "separable_compilation" IN_LIST Kokkos_FIND_COMPONENTS) + # + # if CUDA was enabled, separable compilation was not specified, and current compiler + # cannot compile CUDA, then set the RULE_LAUNCH_COMPILE and RULE_LAUNCH_LINK globally and + # kokkos_launch_compiler will re-direct to the compiler used to compile CUDA code during installation. + # kokkos_launch_compiler will re-direct if ${CMAKE_CXX_COMPILER} and -DKOKKOS_DEPENDENCE is present, + # otherwise, the original command will be executed + # + # run test to see if CMAKE_CXX_COMPILER=nvcc_wrapper kokkos_compiler_is_nvcc(IS_NVCC ${CMAKE_CXX_COMPILER}) - # if not nvcc_wrapper, use RULE_LAUNCH_COMPILE and RULE_LAUNCH_LINK - IF(NOT IS_NVCC AND NOT CMAKE_CXX_COMPILER_ID STREQUAL Clang AND - (NOT DEFINED Kokkos_LAUNCH_COMPILER OR Kokkos_LAUNCH_COMPILER)) - MESSAGE(STATUS "kokkos_launch_compiler is enabled globally. C++ compiler commands with -DKOKKOS_DEPENDENCE will be redirected to nvcc_wrapper") + + # if not nvcc_wrapper and Kokkos_LAUNCH_COMPILER was not set to OFF + IF(NOT IS_NVCC AND (NOT DEFINED Kokkos_LAUNCH_COMPILER OR Kokkos_LAUNCH_COMPILER)) + MESSAGE(STATUS "kokkos_launch_compiler is enabled globally. C++ compiler commands with -DKOKKOS_DEPENDENCE will be redirected to the appropriate compiler for Kokkos") kokkos_compilation(GLOBAL) ENDIF() - UNSET(IS_NVCC) # be mindful of the environment, pollution is bad + + # be mindful of the environment, pollution is bad + UNSET(IS_NVCC) ENDIF() diff --git a/lib/kokkos/cmake/KokkosConfigCommon.cmake.in b/lib/kokkos/cmake/KokkosConfigCommon.cmake.in index 42c755c215..ab93e65afe 100644 --- a/lib/kokkos/cmake/KokkosConfigCommon.cmake.in +++ b/lib/kokkos/cmake/KokkosConfigCommon.cmake.in @@ -3,6 +3,7 @@ SET(Kokkos_OPTIONS @KOKKOS_ENABLED_OPTIONS@) SET(Kokkos_TPLS @KOKKOS_ENABLED_TPLS@) SET(Kokkos_ARCH @KOKKOS_ENABLED_ARCH_LIST@) SET(Kokkos_CXX_COMPILER "@CMAKE_CXX_COMPILER@") +SET(Kokkos_CXX_COMPILER_ID "@KOKKOS_CXX_COMPILER_ID@") # These are needed by KokkosKernels FOREACH(DEV ${Kokkos_DEVICES}) @@ -13,13 +14,13 @@ IF(NOT Kokkos_FIND_QUIETLY) MESSAGE(STATUS "Enabled Kokkos devices: ${Kokkos_DEVICES}") ENDIF() -IF (Kokkos_ENABLE_CUDA AND ${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.14.0") - #If we are building CUDA, we have tricked CMake because we declare a CXX project - #If the default C++ standard for a given compiler matches the requested - #standard, then CMake just omits the -std flag in later versions of CMake - #This breaks CUDA compilation (CUDA compiler can have a different default - #-std then the underlying host compiler by itself). Setting this variable - #forces CMake to always add the -std flag even if it thinks it doesn't need it +IF (Kokkos_ENABLE_CUDA) + # If we are building CUDA, we have tricked CMake because we declare a CXX project + # If the default C++ standard for a given compiler matches the requested + # standard, then CMake just omits the -std flag in later versions of CMake + # This breaks CUDA compilation (CUDA compiler can have a different default + # -std then the underlying host compiler by itself). Setting this variable + # forces CMake to always add the -std flag even if it thinks it doesn't need it SET(CMAKE_CXX_STANDARD_DEFAULT 98 CACHE INTERNAL "" FORCE) ENDIF() @@ -90,52 +91,6 @@ function(kokkos_check) endif() endfunction() -# this function is provided to easily select which files use nvcc_wrapper: -# -# GLOBAL --> all files -# TARGET --> all files in a target -# SOURCE --> specific source files -# DIRECTORY --> all files in directory -# PROJECT --> all files/targets in a project/subproject -# -FUNCTION(kokkos_compilation) - CMAKE_PARSE_ARGUMENTS(COMP "GLOBAL;PROJECT" "" "DIRECTORY;TARGET;SOURCE" ${ARGN}) - - # search relative first and then absolute - SET(_HINTS "${CMAKE_CURRENT_LIST_DIR}/../.." "@CMAKE_INSTALL_PREFIX@") - - # find kokkos_launch_compiler - FIND_PROGRAM(Kokkos_COMPILE_LAUNCHER - NAMES kokkos_launch_compiler - HINTS ${_HINTS} - PATHS ${_HINTS} - PATH_SUFFIXES bin) - - IF(NOT Kokkos_COMPILE_LAUNCHER) - MESSAGE(FATAL_ERROR "Kokkos could not find 'kokkos_launch_compiler'. Please set '-DKokkos_COMPILE_LAUNCHER=/path/to/launcher'") - ENDIF() - - IF(COMP_GLOBAL) - # if global, don't bother setting others - SET_PROPERTY(GLOBAL PROPERTY RULE_LAUNCH_COMPILE "${Kokkos_COMPILE_LAUNCHER} ${CMAKE_CXX_COMPILER}") - SET_PROPERTY(GLOBAL PROPERTY RULE_LAUNCH_LINK "${Kokkos_COMPILE_LAUNCHER} ${CMAKE_CXX_COMPILER}") - ELSE() - FOREACH(_TYPE PROJECT DIRECTORY TARGET SOURCE) - # make project/subproject scoping easy, e.g. KokkosCompilation(PROJECT) after project(...) - IF("${_TYPE}" STREQUAL "PROJECT" AND COMP_${_TYPE}) - LIST(APPEND COMP_DIRECTORY ${PROJECT_SOURCE_DIR}) - UNSET(COMP_${_TYPE}) - ENDIF() - # set the properties if defined - IF(COMP_${_TYPE}) - # MESSAGE(STATUS "Using nvcc_wrapper :: ${_TYPE} :: ${COMP_${_TYPE}}") - SET_PROPERTY(${_TYPE} ${COMP_${_TYPE}} PROPERTY RULE_LAUNCH_COMPILE "${Kokkos_COMPILE_LAUNCHER} ${CMAKE_CXX_COMPILER}") - SET_PROPERTY(${_TYPE} ${COMP_${_TYPE}} PROPERTY RULE_LAUNCH_LINK "${Kokkos_COMPILE_LAUNCHER} ${CMAKE_CXX_COMPILER}") - ENDIF() - ENDFOREACH() - ENDIF() -ENDFUNCTION() - # A test to check whether a downstream project set the C++ compiler to NVCC or not # this is called only when Kokkos was installed with Kokkos_ENABLE_CUDA=ON FUNCTION(kokkos_compiler_is_nvcc VAR COMPILER) @@ -159,3 +114,161 @@ FUNCTION(kokkos_compiler_is_nvcc VAR COMPILER) ENDIF() ENDFUNCTION() +# this function checks whether the current CXX compiler supports building CUDA +FUNCTION(kokkos_cxx_compiler_cuda_test _VAR _COMPILER) + + FILE(WRITE ${PROJECT_BINARY_DIR}/compile_tests/compiles_cuda.cu +" +#include +#include + +__global__ +void kernel(int sz, double* data) +{ + int _beg = blockIdx.x * blockDim.x + threadIdx.x; + for(int i = _beg; i < sz; ++i) + data[i] += static_cast(i); +} + +int main() +{ + double* data = NULL; + int blocks = 64; + int grids = 64; + int ret = cudaMalloc(&data, blocks * grids * sizeof(double)); + if(ret != cudaSuccess) + return EXIT_FAILURE; + kernel<<>>(blocks * grids, data); + cudaDeviceSynchronize(); + return EXIT_SUCCESS; +} +") + + # save the command for debugging + SET(_COMMANDS "${_COMPILER} ${ARGN} -c ${PROJECT_BINARY_DIR}/compile_tests/compiles_cuda.cu") + + # use execute_process instead of try compile because we want to set custom compiler + EXECUTE_PROCESS(COMMAND ${_COMPILER} ${ARGN} -c ${PROJECT_BINARY_DIR}/compile_tests/compiles_cuda.cu + RESULT_VARIABLE _RET + WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/compile_tests + TIMEOUT 15 + OUTPUT_QUIET + ERROR_QUIET) + + IF(NOT _RET EQUAL 0) + # save the command for debugging + SET(_COMMANDS "${_COMMAND}\n${_COMPILER} --cuda-gpu-arch=sm_35 ${ARGN} -c ${PROJECT_BINARY_DIR}/compile_tests/compiles_cuda.cu") + # try the compile test again with clang arguments + EXECUTE_PROCESS(COMMAND ${_COMPILER} --cuda-gpu-arch=sm_35 -c ${PROJECT_BINARY_DIR}/compile_tests/compiles_cuda.cu + RESULT_VARIABLE _RET + WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/compile_tests + TIMEOUT 15 + OUTPUT_QUIET + ERROR_QUIET) + ENDIF() + + SET(${_VAR}_COMMANDS "${_COMMANDS}" PARENT_SCOPE) + SET(${_VAR} ${_RET} PARENT_SCOPE) +ENDFUNCTION() + +# this function is provided to easily select which files use the same compiler as Kokkos +# when it was installed (or nvcc_wrapper): +# +# GLOBAL --> all files +# TARGET --> all files in a target +# SOURCE --> specific source files +# DIRECTORY --> all files in directory +# PROJECT --> all files/targets in a project/subproject +# +# Use the COMPILER argument to specify a compiler, if needed. By default, it will +# set the values to ${Kokkos_CXX_COMPILER} unless Kokkos_ENABLE_CUDA=ON and +# Kokkos_CXX_COMPILER_ID is NVIDIA, then it will set it to nvcc_wrapper +# +# Use CHECK_CUDA_COMPILES to run a check when CUDA is enabled +# +FUNCTION(kokkos_compilation) + CMAKE_PARSE_ARGUMENTS(COMP + "GLOBAL;PROJECT;CHECK_CUDA_COMPILES" + "COMPILER" + "DIRECTORY;TARGET;SOURCE;COMMAND_PREFIX" + ${ARGN}) + + # if built w/o CUDA support, we want to basically make this a no-op + SET(_Kokkos_ENABLE_CUDA @Kokkos_ENABLE_CUDA@) + + # search relative first and then absolute + SET(_HINTS "${CMAKE_CURRENT_LIST_DIR}/../.." "@CMAKE_INSTALL_PREFIX@") + + # find kokkos_launch_compiler + FIND_PROGRAM(Kokkos_COMPILE_LAUNCHER + NAMES kokkos_launch_compiler + HINTS ${_HINTS} + PATHS ${_HINTS} + PATH_SUFFIXES bin) + + IF(NOT Kokkos_COMPILE_LAUNCHER) + MESSAGE(FATAL_ERROR "Kokkos could not find 'kokkos_launch_compiler'. Please set '-DKokkos_COMPILE_LAUNCHER=/path/to/launcher'") + ENDIF() + + # if COMPILER was not specified, assume Kokkos_CXX_COMPILER + IF(NOT COMP_COMPILER) + SET(COMP_COMPILER ${Kokkos_CXX_COMPILER}) + IF(_Kokkos_ENABLE_CUDA AND Kokkos_CXX_COMPILER_ID STREQUAL NVIDIA) + # find nvcc_wrapper + FIND_PROGRAM(Kokkos_NVCC_WRAPPER + NAMES nvcc_wrapper + HINTS ${_HINTS} + PATHS ${_HINTS} + PATH_SUFFIXES bin) + # fatal if we can't nvcc_wrapper + IF(NOT Kokkos_NVCC_WRAPPER) + MESSAGE(FATAL_ERROR "Kokkos could not find nvcc_wrapper. Please set '-DKokkos_NVCC_WRAPPER=/path/to/nvcc_wrapper'") + ENDIF() + SET(COMP_COMPILER ${Kokkos_NVCC_WRAPPER}) + ENDIF() + ENDIF() + + # check that the original compiler still exists! + IF(NOT EXISTS ${COMP_COMPILER}) + MESSAGE(FATAL_ERROR "Kokkos could not find original compiler: '${COMP_COMPILER}'") + ENDIF() + + # try to ensure that compiling cuda code works! + IF(_Kokkos_ENABLE_CUDA AND COMP_CHECK_CUDA_COMPILES) + + # this may fail if kokkos_compiler launcher was used during install + kokkos_cxx_compiler_cuda_test(_COMPILES_CUDA + ${Kokkos_COMPILE_LAUNCHER} ${COMP_COMPILER} ${CMAKE_CXX_COMPILER}) + + # if above failed, throw an error + IF(NOT _COMPILES_CUDA) + MESSAGE(FATAL_ERROR "kokkos_cxx_compiler_cuda_test failed! Test commands:\n${_COMPILES_CUDA_COMMANDS}") + ENDIF() + ENDIF() + + IF(COMP_COMMAND_PREFIX) + SET(_PREFIX "${COMP_COMMAND_PREFIX}") + STRING(REPLACE ";" " " _PREFIX "${COMP_COMMAND_PREFIX}") + SET(Kokkos_COMPILER_LAUNCHER "${_PREFIX} ${Kokkos_COMPILE_LAUNCHER}") + ENDIF() + + IF(COMP_GLOBAL) + # if global, don't bother setting others + SET_PROPERTY(GLOBAL PROPERTY RULE_LAUNCH_COMPILE "${Kokkos_COMPILE_LAUNCHER} ${COMP_COMPILER} ${CMAKE_CXX_COMPILER}") + SET_PROPERTY(GLOBAL PROPERTY RULE_LAUNCH_LINK "${Kokkos_COMPILE_LAUNCHER} ${COMP_COMPILER} ${CMAKE_CXX_COMPILER}") + ELSE() + FOREACH(_TYPE PROJECT DIRECTORY TARGET SOURCE) + # make project/subproject scoping easy, e.g. KokkosCompilation(PROJECT) after project(...) + IF("${_TYPE}" STREQUAL "PROJECT" AND COMP_${_TYPE}) + LIST(APPEND COMP_DIRECTORY ${PROJECT_SOURCE_DIR}) + UNSET(COMP_${_TYPE}) + ENDIF() + # set the properties if defined + IF(COMP_${_TYPE}) + # MESSAGE(STATUS "Using ${COMP_COMPILER} :: ${_TYPE} :: ${COMP_${_TYPE}}") + SET_PROPERTY(${_TYPE} ${COMP_${_TYPE}} PROPERTY RULE_LAUNCH_COMPILE "${Kokkos_COMPILE_LAUNCHER} ${COMP_COMPILER} ${CMAKE_CXX_COMPILER}") + SET_PROPERTY(${_TYPE} ${COMP_${_TYPE}} PROPERTY RULE_LAUNCH_LINK "${Kokkos_COMPILE_LAUNCHER} ${COMP_COMPILER} ${CMAKE_CXX_COMPILER}") + ENDIF() + ENDFOREACH() + ENDIF() +ENDFUNCTION() diff --git a/lib/kokkos/cmake/KokkosCore_config.h.in b/lib/kokkos/cmake/KokkosCore_config.h.in index 0259fe69d5..fbfae3711e 100644 --- a/lib/kokkos/cmake/KokkosCore_config.h.in +++ b/lib/kokkos/cmake/KokkosCore_config.h.in @@ -78,6 +78,7 @@ #cmakedefine KOKKOS_ARCH_POWER7 #cmakedefine KOKKOS_ARCH_POWER8 #cmakedefine KOKKOS_ARCH_POWER9 +#cmakedefine KOKKOS_ARCH_INTEL_GEN #cmakedefine KOKKOS_ARCH_KEPLER #cmakedefine KOKKOS_ARCH_KEPLER30 #cmakedefine KOKKOS_ARCH_KEPLER32 @@ -95,5 +96,8 @@ #cmakedefine KOKKOS_ARCH_VOLTA72 #cmakedefine KOKKOS_ARCH_TURING75 #cmakedefine KOKKOS_ARCH_AMPERE80 +#cmakedefine KOKKOS_ARCH_AMPERE86 #cmakedefine KOKKOS_ARCH_AMD_ZEN #cmakedefine KOKKOS_ARCH_AMD_ZEN2 + +#cmakedefine KOKKOS_IMPL_DISABLE_SYCL_DEVICE_PRINTF diff --git a/lib/kokkos/cmake/Modules/CudaToolkit.cmake b/lib/kokkos/cmake/Modules/CudaToolkit.cmake index d620a71d36..eda5541f7c 100644 --- a/lib/kokkos/cmake/Modules/CudaToolkit.cmake +++ b/lib/kokkos/cmake/Modules/CudaToolkit.cmake @@ -481,76 +481,6 @@ if(CMAKE_CUDA_COMPILER_LOADED AND NOT CUDAToolkit_BIN_DIR AND CMAKE_CUDA_COMPILE unset(cuda_dir) endif() -IF(CMAKE_VERSION VERSION_LESS "3.12.0") - function(import_target_link_libraries target) - cmake_parse_arguments(HACK - "SYSTEM;INTERFACE;PUBLIC" - "" - "" - ${ARGN} - ) - get_target_property(LIBS ${target} INTERFACE_LINK_LIBRARIES) - if (LIBS) - list(APPEND LIBS ${HACK_UNPARSED_ARGUMENTS}) - else() - set(LIBS ${HACK_UNPARSED_ARGUMENTS}) - endif() - set_target_properties(${target} PROPERTIES - INTERFACE_LINK_LIBRARIES "${LIBS}") - endfunction() -ELSE() - function(import_target_link_libraries) - target_link_libraries(${ARGN}) - endfunction() -ENDIF() - -IF(CMAKE_VERSION VERSION_LESS "3.13.0") - function(import_target_link_directories target) - cmake_parse_arguments(HACK - "SYSTEM;INTERFACE;PUBLIC" - "" - "" - ${ARGN} - ) - get_target_property(LINK_LIBS ${target} INTERFACE_LINK_LIBRARIES) - if (LINK_LIBS) #could be not-found - set(LINK_LIBS_LIST ${LINK_LIBS}) - endif() - foreach(LIB ${HACK_UNPARSED_ARGUMENTS}) - list(APPEND LINK_LIBS_LIST -L${LIB}) - endforeach() - set_target_properties(${target} PROPERTIES - INTERFACE_LINK_LIBRARIES "${LINK_LIBS_LIST}") - endfunction() -ELSE() - function(import_target_link_directories) - target_link_directories(${ARGN}) - endfunction() -ENDIF() - -IF(CMAKE_VERSION VERSION_LESS "3.12.0") - function(import_target_include_directories target) - cmake_parse_arguments(HACK - "SYSTEM;INTERFACE;PUBLIC" - "" - "" - ${ARGN} - ) - get_target_property(INLUDE_DIRS ${target} INTERFACE_INCLUDE_DIRECTORIES) - if (INCLUDE_DIRS) - list(APPEND INCLUDE_DIRS ${HACK_UNPARSED_ARGUMENTS}) - else() - set(INCLUDE_DIRS ${HACK_UNPARSED_ARGUMENTS}) - endif() - set_target_properties(${target} PROPERTIES - INTERFACE_INCLUDE_DIRECTORIES "${INCLUDE_DIRS}") - endfunction() -ELSE() - function(import_target_include_directories) - target_include_directories(${ARGN}) - endfunction() -ENDIF() - # Try language- or user-provided path first. if(CUDAToolkit_BIN_DIR) find_program(CUDAToolkit_NVCC_EXECUTABLE @@ -854,11 +784,11 @@ if(CUDAToolkit_FOUND) if (NOT TARGET CUDA::${lib_name} AND CUDA_${lib_name}_LIBRARY) add_library(CUDA::${lib_name} IMPORTED INTERFACE) - import_target_include_directories(CUDA::${lib_name} SYSTEM INTERFACE "${CUDAToolkit_INCLUDE_DIRS}") - import_target_link_libraries(CUDA::${lib_name} INTERFACE "${CUDA_${lib_name}_LIBRARY}") + target_include_directories(CUDA::${lib_name} SYSTEM INTERFACE "${CUDAToolkit_INCLUDE_DIRS}") + target_link_libraries(CUDA::${lib_name} INTERFACE "${CUDA_${lib_name}_LIBRARY}") foreach(dep ${arg_DEPS}) if(TARGET CUDA::${dep}) - import_target_link_libraries(CUDA::${lib_name} INTERFACE CUDA::${dep}) + target_link_libraries(CUDA::${lib_name} INTERFACE CUDA::${dep}) endif() endforeach() endif() @@ -866,8 +796,8 @@ if(CUDAToolkit_FOUND) if(NOT TARGET CUDA::toolkit) add_library(CUDA::toolkit IMPORTED INTERFACE) - import_target_include_directories(CUDA::toolkit SYSTEM INTERFACE "${CUDAToolkit_INCLUDE_DIRS}") - import_target_link_directories(CUDA::toolkit INTERFACE "${CUDAToolkit_LIBRARY_DIR}") + target_include_directories(CUDA::toolkit SYSTEM INTERFACE "${CUDAToolkit_INCLUDE_DIRS}") + target_link_directories(CUDA::toolkit INTERFACE "${CUDAToolkit_LIBRARY_DIR}") endif() _CUDAToolkit_find_and_add_import_lib(cuda_driver ALT cuda) @@ -882,11 +812,11 @@ if(CUDAToolkit_FOUND) AND TARGET CUDA::cudart_static) add_library(CUDA::cudart_static_deps IMPORTED INTERFACE) - import_target_link_libraries(CUDA::cudart_static INTERFACE CUDA::cudart_static_deps) + target_link_libraries(CUDA::cudart_static INTERFACE CUDA::cudart_static_deps) if(UNIX AND (CMAKE_C_COMPILER OR CMAKE_CXX_COMPILER)) find_package(Threads REQUIRED) - import_target_link_libraries(CUDA::cudart_static_deps INTERFACE Threads::Threads ${CMAKE_DL_LIBS}) + target_link_libraries(CUDA::cudart_static_deps INTERFACE Threads::Threads ${CMAKE_DL_LIBS}) endif() if(UNIX AND NOT APPLE) @@ -896,7 +826,7 @@ if(CUDAToolkit_FOUND) if(NOT CUDAToolkit_rt_LIBRARY) message(WARNING "Could not find librt library, needed by CUDA::cudart_static") else() - import_target_link_libraries(CUDA::cudart_static_deps INTERFACE ${CUDAToolkit_rt_LIBRARY}) + target_link_libraries(CUDA::cudart_static_deps INTERFACE ${CUDAToolkit_rt_LIBRARY}) endif() endif() endif() diff --git a/lib/kokkos/cmake/Modules/FindTPLCUDA.cmake b/lib/kokkos/cmake/Modules/FindTPLCUDA.cmake index a1072a60c6..8d58d96415 100644 --- a/lib/kokkos/cmake/Modules/FindTPLCUDA.cmake +++ b/lib/kokkos/cmake/Modules/FindTPLCUDA.cmake @@ -25,7 +25,7 @@ IF (TARGET CUDA::cuda_driver) SET(FOUND_CUDA_DRIVER TRUE) KOKKOS_EXPORT_IMPORTED_TPL(CUDA::cuda_driver) ELSE() - SET(FOUND_CUDA_DRIVVER FALSE) + SET(FOUND_CUDA_DRIVER FALSE) ENDIF() include(FindPackageHandleStandardArgs) diff --git a/lib/kokkos/cmake/Modules/FindTPLPTHREAD.cmake b/lib/kokkos/cmake/Modules/FindTPLPTHREAD.cmake index 1d154e29af..a743fca0e4 100644 --- a/lib/kokkos/cmake/Modules/FindTPLPTHREAD.cmake +++ b/lib/kokkos/cmake/Modules/FindTPLPTHREAD.cmake @@ -10,7 +10,7 @@ TRY_COMPILE(KOKKOS_HAS_PTHREAD_ARG # ${CMAKE_CXX${KOKKOS_CXX_STANDARD}_STANDARD_COMPILE_OPTION} INCLUDE(FindPackageHandleStandardArgs) -FIND_PACKAGE_HANDLE_STANDARD_ARGS(PTHREAD DEFAULT_MSG KOKKOS_HAS_PTHREAD_ARG) +FIND_PACKAGE_HANDLE_STANDARD_ARGS(TPLPTHREAD DEFAULT_MSG KOKKOS_HAS_PTHREAD_ARG) #Only create the TPL if we succeed IF (KOKKOS_HAS_PTHREAD_ARG) KOKKOS_CREATE_IMPORTED_TPL(PTHREAD diff --git a/lib/kokkos/cmake/Modules/FindTPLROCM.cmake b/lib/kokkos/cmake/Modules/FindTPLROCM.cmake new file mode 100644 index 0000000000..512ad6ceb2 --- /dev/null +++ b/lib/kokkos/cmake/Modules/FindTPLROCM.cmake @@ -0,0 +1,11 @@ +include(FindPackageHandleStandardArgs) + +FIND_LIBRARY(AMD_HIP_LIBRARY amdhip64 PATHS ENV ROCM_PATH PATH_SUFFIXES lib) +FIND_LIBRARY(HSA_RUNTIME_LIBRARY hsa-runtime64 PATHS ENV ROCM_PATH PATH_SUFFIXES lib) + +find_package_handle_standard_args(TPLROCM DEFAULT_MSG AMD_HIP_LIBRARY HSA_RUNTIME_LIBRARY) + +kokkos_create_imported_tpl(ROCM INTERFACE + LINK_LIBRARIES ${HSA_RUNTIME_LIBRARY} ${AMD_HIP_LIBRARY} + COMPILE_DEFINITIONS __HIP_ROCclr__ +) diff --git a/lib/kokkos/cmake/compile_tests/cplusplus14.cpp b/lib/kokkos/cmake/compile_tests/cplusplus14.cpp new file mode 100644 index 0000000000..52ec9885ec --- /dev/null +++ b/lib/kokkos/cmake/compile_tests/cplusplus14.cpp @@ -0,0 +1,8 @@ +#include + +int main() { + // _t versions of type traits were added in C++14 + std::remove_cv_t i = 0; + + return i; +} diff --git a/lib/kokkos/cmake/compile_tests/cuda_compute_capability.cc b/lib/kokkos/cmake/compile_tests/cuda_compute_capability.cc index 48c01c070c..a26ac5af4b 100644 --- a/lib/kokkos/cmake/compile_tests/cuda_compute_capability.cc +++ b/lib/kokkos/cmake/compile_tests/cuda_compute_capability.cc @@ -72,6 +72,7 @@ int main() { case 72: std::cout << "Set -DKokkos_ARCH_VOLTA72=ON ." << std::endl; break; case 75: std::cout << "Set -DKokkos_ARCH_TURING75=ON ." << std::endl; break; case 80: std::cout << "Set -DKokkos_ARCH_AMPERE80=ON ." << std::endl; break; + case 86: std::cout << "Set -DKokkos_ARCH_AMPERE86=ON ." << std::endl; break; default: std::cout << "Compute capability " << compute_capability << " is not supported" << std::endl; diff --git a/lib/kokkos/cmake/compile_tests/pthread.cpp b/lib/kokkos/cmake/compile_tests/pthread.cpp index 92310da029..3f83bf6a5f 100644 --- a/lib/kokkos/cmake/compile_tests/pthread.cpp +++ b/lib/kokkos/cmake/compile_tests/pthread.cpp @@ -2,7 +2,7 @@ void* kokkos_test(void* args) { return args; } -int main(void) { +int main() { pthread_t thread; /* Use NULL to avoid C++11. Some compilers do not have C++11 by default. Forcing C++11 diff --git a/lib/kokkos/cmake/fake_tribits.cmake b/lib/kokkos/cmake/fake_tribits.cmake index 2e82a46235..fbd6745a60 100644 --- a/lib/kokkos/cmake/fake_tribits.cmake +++ b/lib/kokkos/cmake/fake_tribits.cmake @@ -81,10 +81,16 @@ ENDMACRO() FUNCTION(KOKKOS_ADD_TEST) if (KOKKOS_HAS_TRILINOS) CMAKE_PARSE_ARGUMENTS(TEST - "" + "SKIP_TRIBITS" "EXE;NAME;TOOL" "ARGS" ${ARGN}) + + IF(TEST_SKIP_TRIBITS) + MESSAGE(STATUS "Skipping test ${TEST_NAME} in TriBits") + RETURN() + ENDIF() + IF(TEST_EXE) SET(EXE_ROOT ${TEST_EXE}) ELSE() @@ -119,11 +125,10 @@ FUNCTION(KOKKOS_ADD_TEST) endif() else() CMAKE_PARSE_ARGUMENTS(TEST - "WILL_FAIL" + "WILL_FAIL;SKIP_TRIBITS" "FAIL_REGULAR_EXPRESSION;PASS_REGULAR_EXPRESSION;EXE;NAME;TOOL" "CATEGORIES;ARGS" ${ARGN}) - SET(TESTS_ADDED) # To match Tribits, we should always be receiving # the root names of exes/libs IF(TEST_EXE) @@ -135,48 +140,27 @@ FUNCTION(KOKKOS_ADD_TEST) # These should be the full target name SET(TEST_NAME ${PACKAGE_NAME}_${TEST_NAME}) SET(EXE ${PACKAGE_NAME}_${EXE_ROOT}) - IF (TEST_ARGS) - SET(TEST_NUMBER 0) - FOREACH (ARG_STR ${TEST_ARGS}) - # This is passed as a single string blob to match TriBITS behavior - # We need this to be turned into a list - STRING(REPLACE " " ";" ARG_STR_LIST ${ARG_STR}) - IF(WIN32) - ADD_TEST(NAME ${TEST_NAME}${TEST_NUMBER} WORKING_DIRECTORY ${LIBRARY_OUTPUT_PATH} - COMMAND ${EXE}${CMAKE_EXECUTABLE_SUFFIX} ${ARG_STR_LIST}) - ELSE() - ADD_TEST(NAME ${TEST_NAME}${TEST_NUMBER} COMMAND ${EXE} ${ARG_STR_LIST}) - ENDIF() - LIST(APPEND TESTS_ADDED "${TEST_NAME}${TEST_NUMBER}") - MATH(EXPR TEST_NUMBER "${TEST_NUMBER} + 1") - ENDFOREACH() + IF(WIN32) + ADD_TEST(NAME ${TEST_NAME} WORKING_DIRECTORY ${LIBRARY_OUTPUT_PATH} + COMMAND ${EXE}${CMAKE_EXECUTABLE_SUFFIX} ${TEST_ARGS}) ELSE() - IF(WIN32) - ADD_TEST(NAME ${TEST_NAME} WORKING_DIRECTORY ${LIBRARY_OUTPUT_PATH} - COMMAND ${EXE}${CMAKE_EXECUTABLE_SUFFIX}) - ELSE() - ADD_TEST(NAME ${TEST_NAME} COMMAND ${EXE}) - ENDIF() - LIST(APPEND TESTS_ADDED "${TEST_NAME}") + ADD_TEST(NAME ${TEST_NAME} COMMAND ${EXE} ${TEST_ARGS}) + ENDIF() + IF(TEST_WILL_FAIL) + SET_TESTS_PROPERTIES(${TEST_NAME} PROPERTIES WILL_FAIL ${TEST_WILL_FAIL}) + ENDIF() + IF(TEST_FAIL_REGULAR_EXPRESSION) + SET_TESTS_PROPERTIES(${TEST_NAME} PROPERTIES FAIL_REGULAR_EXPRESSION ${TEST_FAIL_REGULAR_EXPRESSION}) + ENDIF() + IF(TEST_PASS_REGULAR_EXPRESSION) + SET_TESTS_PROPERTIES(${TEST_NAME} PROPERTIES PASS_REGULAR_EXPRESSION ${TEST_PASS_REGULAR_EXPRESSION}) + ENDIF() + IF(TEST_TOOL) + ADD_DEPENDENCIES(${EXE} ${TEST_TOOL}) #make sure the exe has to build the tool + SET_PROPERTY(TEST ${TEST_NAME} APPEND_STRING PROPERTY ENVIRONMENT "KOKKOS_PROFILE_LIBRARY=$") ENDIF() - - FOREACH(TEST_NAME ${TESTS_ADDED}) - IF(TEST_WILL_FAIL) - SET_TESTS_PROPERTIES(${TEST_NAME} PROPERTIES WILL_FAIL ${TEST_WILL_FAIL}) - ENDIF() - IF(TEST_FAIL_REGULAR_EXPRESSION) - SET_TESTS_PROPERTIES(${TEST_NAME} PROPERTIES FAIL_REGULAR_EXPRESSION ${TEST_FAIL_REGULAR_EXPRESSION}) - ENDIF() - IF(TEST_PASS_REGULAR_EXPRESSION) - SET_TESTS_PROPERTIES(${TEST_NAME} PROPERTIES PASS_REGULAR_EXPRESSION ${TEST_PASS_REGULAR_EXPRESSION}) - ENDIF() - if(TEST_TOOL) - add_dependencies(${EXE} ${TEST_TOOL}) #make sure the exe has to build the tool - set_property(TEST ${TEST_NAME} APPEND_STRING PROPERTY ENVIRONMENT "KOKKOS_PROFILE_LIBRARY=$") - endif() - ENDFOREACH() VERIFY_EMPTY(KOKKOS_ADD_TEST ${TEST_UNPARSED_ARGUMENTS}) - endif() + ENDIF() ENDFUNCTION() FUNCTION(KOKKOS_ADD_ADVANCED_TEST) @@ -326,14 +310,6 @@ ENDIF() ENDFUNCTION() -FUNCTION(KOKKOS_TARGET_COMPILE_DEFINITIONS) - IF (KOKKOS_HAS_TRILINOS) - TARGET_COMPILE_DEFINITIONS(${TARGET} ${ARGN}) - ELSE() - TARGET_COMPILE_DEFINITIONS(${TARGET} ${ARGN}) - ENDIF() -ENDFUNCTION() - FUNCTION(KOKKOS_INCLUDE_DIRECTORIES) IF(KOKKOS_HAS_TRILINOS) TRIBITS_INCLUDE_DIRECTORIES(${ARGN}) @@ -350,10 +326,6 @@ ENDIF() ENDFUNCTION() -MACRO(KOKKOS_ADD_COMPILE_OPTIONS) -ADD_COMPILE_OPTIONS(${ARGN}) -ENDMACRO() - MACRO(PRINTALL match) get_cmake_property(_variableNames VARIABLES) list (SORT _variableNames) @@ -376,4 +348,3 @@ FUNCTION(GLOBAL_APPEND VARNAME) LIST(APPEND TEMP ${ARGN}) GLOBAL_SET(${VARNAME} ${TEMP}) ENDFUNCTION() - diff --git a/lib/kokkos/cmake/kokkos_arch.cmake b/lib/kokkos/cmake/kokkos_arch.cmake index 53aaf7dccf..ec18e70a36 100644 --- a/lib/kokkos/cmake/kokkos_arch.cmake +++ b/lib/kokkos/cmake/kokkos_arch.cmake @@ -35,7 +35,7 @@ KOKKOS_ARCH_OPTION(ARMV80 HOST "ARMv8.0 Compatible CPU") KOKKOS_ARCH_OPTION(ARMV81 HOST "ARMv8.1 Compatible CPU") KOKKOS_ARCH_OPTION(ARMV8_THUNDERX HOST "ARMv8 Cavium ThunderX CPU") KOKKOS_ARCH_OPTION(ARMV8_THUNDERX2 HOST "ARMv8 Cavium ThunderX2 CPU") -KOKKOS_ARCH_OPTION(A64FX HOST "ARMv8.2 with SVE Suport") +KOKKOS_ARCH_OPTION(A64FX HOST "ARMv8.2 with SVE Support") KOKKOS_ARCH_OPTION(WSM HOST "Intel Westmere CPU") KOKKOS_ARCH_OPTION(SNB HOST "Intel Sandy/Ivy Bridge CPUs") KOKKOS_ARCH_OPTION(HSW HOST "Intel Haswell CPUs") @@ -60,11 +60,12 @@ KOKKOS_ARCH_OPTION(VOLTA70 GPU "NVIDIA Volta generation CC 7.0") KOKKOS_ARCH_OPTION(VOLTA72 GPU "NVIDIA Volta generation CC 7.2") KOKKOS_ARCH_OPTION(TURING75 GPU "NVIDIA Turing generation CC 7.5") KOKKOS_ARCH_OPTION(AMPERE80 GPU "NVIDIA Ampere generation CC 8.0") +KOKKOS_ARCH_OPTION(AMPERE86 GPU "NVIDIA Ampere generation CC 8.6") KOKKOS_ARCH_OPTION(ZEN HOST "AMD Zen architecture") KOKKOS_ARCH_OPTION(ZEN2 HOST "AMD Zen2 architecture") KOKKOS_ARCH_OPTION(VEGA900 GPU "AMD GPU MI25 GFX900") KOKKOS_ARCH_OPTION(VEGA906 GPU "AMD GPU MI50/MI60 GFX906") -KOKKOS_ARCH_OPTION(VEGA908 GPU "AMD GPU") +KOKKOS_ARCH_OPTION(VEGA908 GPU "AMD GPU MI100 GFX908") KOKKOS_ARCH_OPTION(INTEL_GEN GPU "Intel GPUs Gen9+") @@ -141,8 +142,16 @@ ENDIF() #------------------------------- KOKKOS_HIP_OPTIONS --------------------------- #clear anything that might be in the cache GLOBAL_SET(KOKKOS_AMDGPU_OPTIONS) -IF(KOKKOS_CXX_COMPILER_ID STREQUAL HIP) - SET(AMDGPU_ARCH_FLAG "--amdgpu-target") +IF(KOKKOS_ENABLE_HIP) + IF(KOKKOS_CXX_COMPILER_ID STREQUAL HIPCC) + SET(AMDGPU_ARCH_FLAG "--amdgpu-target") + ELSE() + SET(AMDGPU_ARCH_FLAG "--offload-arch") + GLOBAL_APPEND(KOKKOS_AMDGPU_OPTIONS -x hip) + IF(DEFINED ENV{ROCM_PATH}) + GLOBAL_APPEND(KOKKOS_AMDGPU_OPTIONS --rocm-path=$ENV{ROCM_PATH}) + ENDIF() + ENDIF() ENDIF() @@ -183,6 +192,8 @@ ENDIF() IF (KOKKOS_ARCH_A64FX) COMPILER_SPECIFIC_FLAGS( DEFAULT -march=armv8.2-a+sve + Clang -march=armv8.2-a+sve -msve-vector-bits=512 + GCC -march=armv8.2-a+sve -msve-vector-bits=512 ) ENDIF() @@ -309,7 +320,7 @@ IF (KOKKOS_ARCH_POWER8 OR KOKKOS_ARCH_POWER9) SET(KOKKOS_USE_ISA_POWERPCLE ON) ENDIF() -IF (Kokkos_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE) +IF (KOKKOS_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE) COMPILER_SPECIFIC_FLAGS( Clang -fcuda-rdc NVIDIA --relocatable-device-code=true @@ -333,8 +344,8 @@ ENDIF() #Right now we cannot get the compiler ID when cross-compiling, so just check #that HIP is enabled -IF (Kokkos_ENABLE_HIP) - IF (Kokkos_ENABLE_HIP_RELOCATABLE_DEVICE_CODE) +IF (KOKKOS_ENABLE_HIP) + IF (KOKKOS_ENABLE_HIP_RELOCATABLE_DEVICE_CODE) COMPILER_SPECIFIC_FLAGS( DEFAULT -fgpu-rdc ) @@ -345,8 +356,7 @@ IF (Kokkos_ENABLE_HIP) ENDIF() ENDIF() - -IF (Kokkos_ENABLE_SYCL) +IF (KOKKOS_ENABLE_SYCL) COMPILER_SPECIFIC_FLAGS( DEFAULT -fsycl ) @@ -363,7 +373,7 @@ FUNCTION(CHECK_CUDA_ARCH ARCH FLAG) MESSAGE(FATAL_ERROR "Multiple GPU architectures given! Already have ${CUDA_ARCH_ALREADY_SPECIFIED}, but trying to add ${ARCH}. If you are re-running CMake, try clearing the cache and running again.") ENDIF() SET(CUDA_ARCH_ALREADY_SPECIFIED ${ARCH} PARENT_SCOPE) - IF (NOT KOKKOS_ENABLE_CUDA AND NOT KOKKOS_ENABLE_OPENMPTARGET) + IF (NOT KOKKOS_ENABLE_CUDA AND NOT KOKKOS_ENABLE_OPENMPTARGET AND NOT KOKKOS_ENABLE_SYCL) MESSAGE(WARNING "Given CUDA arch ${ARCH}, but Kokkos_ENABLE_CUDA and Kokkos_ENABLE_OPENMPTARGET are OFF. Option will be ignored.") UNSET(KOKKOS_ARCH_${ARCH} PARENT_SCOPE) ELSE() @@ -396,6 +406,7 @@ CHECK_CUDA_ARCH(VOLTA70 sm_70) CHECK_CUDA_ARCH(VOLTA72 sm_72) CHECK_CUDA_ARCH(TURING75 sm_75) CHECK_CUDA_ARCH(AMPERE80 sm_80) +CHECK_CUDA_ARCH(AMPERE86 sm_86) SET(AMDGPU_ARCH_ALREADY_SPECIFIED "") FUNCTION(CHECK_AMDGPU_ARCH ARCH FLAG) @@ -405,12 +416,12 @@ FUNCTION(CHECK_AMDGPU_ARCH ARCH FLAG) ENDIF() SET(AMDGPU_ARCH_ALREADY_SPECIFIED ${ARCH} PARENT_SCOPE) IF (NOT KOKKOS_ENABLE_HIP AND NOT KOKKOS_ENABLE_OPENMPTARGET) - MESSAGE(WARNING "Given HIP arch ${ARCH}, but Kokkos_ENABLE_AMDGPU and Kokkos_ENABLE_OPENMPTARGET are OFF. Option will be ignored.") + MESSAGE(WARNING "Given AMD GPU architecture ${ARCH}, but Kokkos_ENABLE_HIP and Kokkos_ENABLE_OPENMPTARGET are OFF. Option will be ignored.") UNSET(KOKKOS_ARCH_${ARCH} PARENT_SCOPE) ELSE() SET(KOKKOS_AMDGPU_ARCH_FLAG ${FLAG} PARENT_SCOPE) GLOBAL_APPEND(KOKKOS_AMDGPU_OPTIONS "${AMDGPU_ARCH_FLAG}=${FLAG}") - IF(KOKKOS_ENABLE_HIP) + IF(KOKKOS_ENABLE_HIP_RELOCATABLE_DEVICE_CODE) GLOBAL_APPEND(KOKKOS_LINK_OPTIONS "${AMDGPU_ARCH_FLAG}=${FLAG}") ENDIF() ENDIF() @@ -451,6 +462,24 @@ IF (KOKKOS_ENABLE_OPENMPTARGET) ENDIF() ENDIF() +IF (KOKKOS_ENABLE_SYCL) + IF(CUDA_ARCH_ALREADY_SPECIFIED) + IF(KOKKOS_ENABLE_UNSUPPORTED_ARCHS) + COMPILER_SPECIFIC_FLAGS( + DEFAULT -fsycl-targets=nvptx64-nvidia-cuda-sycldevice + ) + # FIXME_SYCL The CUDA backend doesn't support printf yet. + GLOBAL_SET(KOKKOS_IMPL_DISABLE_SYCL_DEVICE_PRINTF ON) + ELSE() + MESSAGE(SEND_ERROR "Setting a CUDA architecture for SYCL is only allowed with Kokkos_ENABLE_UNSUPPORTED_ARCHS=ON!") + ENDIF() + ELSEIF(KOKKOS_ARCH_INTEL_GEN) + COMPILER_SPECIFIC_FLAGS( + DEFAULT -fsycl-targets=spir64_gen-unknown-unknown-sycldevice -Xsycl-target-backend "-device skl" + ) + ENDIF() +ENDIF() + IF(KOKKOS_ENABLE_CUDA AND NOT CUDA_ARCH_ALREADY_SPECIFIED) # Try to autodetect the CUDA Compute Capability by asking the device SET(_BINARY_TEST_DIR ${CMAKE_CURRENT_BINARY_DIR}/cmake/compile_tests/CUDAComputeCapabilityWorkdir) @@ -464,6 +493,43 @@ IF(KOKKOS_ENABLE_CUDA AND NOT CUDA_ARCH_ALREADY_SPECIFIED) ${CMAKE_CURRENT_SOURCE_DIR}/cmake/compile_tests/cuda_compute_capability.cc COMPILE_DEFINITIONS -DSM_ONLY RUN_OUTPUT_VARIABLE _CUDA_COMPUTE_CAPABILITY) + + # if user is using kokkos_compiler_launcher, above will fail. + IF(NOT _COMPILE_RESULT OR NOT _RESULT EQUAL 0) + # check to see if CUDA is not already enabled (may happen when Kokkos is subproject) + GET_PROPERTY(_ENABLED_LANGUAGES GLOBAL PROPERTY ENABLED_LANGUAGES) + # language has to be fully enabled, just checking for CMAKE_CUDA_COMPILER isn't enough + IF(NOT "CUDA" IN_LIST _ENABLED_LANGUAGES) + # make sure the user knows that we aren't using CUDA compiler for anything else + MESSAGE(STATUS "CUDA auto-detection of architecture failed with ${CMAKE_CXX_COMPILER}. Enabling CUDA language ONLY to auto-detect architecture...") + INCLUDE(CheckLanguage) + CHECK_LANGUAGE(CUDA) + IF(CMAKE_CUDA_COMPILER) + ENABLE_LANGUAGE(CUDA) + ELSE() + MESSAGE(STATUS "CUDA language could not be enabled") + ENDIF() + ENDIF() + + # if CUDA was enabled, this will be defined + IF(CMAKE_CUDA_COMPILER) + # copy our test to .cu so cmake compiles as CUDA + CONFIGURE_FILE( + ${PROJECT_SOURCE_DIR}/cmake/compile_tests/cuda_compute_capability.cc + ${PROJECT_BINARY_DIR}/compile_tests/cuda_compute_capability.cu + COPYONLY + ) + # run test again + TRY_RUN( + _RESULT + _COMPILE_RESULT + ${_BINARY_TEST_DIR} + ${PROJECT_BINARY_DIR}/compile_tests/cuda_compute_capability.cu + COMPILE_DEFINITIONS -DSM_ONLY + RUN_OUTPUT_VARIABLE _CUDA_COMPUTE_CAPABILITY) + ENDIF() + ENDIF() + LIST(FIND KOKKOS_CUDA_ARCH_FLAGS sm_${_CUDA_COMPUTE_CAPABILITY} FLAG_INDEX) IF(_COMPILE_RESULT AND _RESULT EQUAL 0 AND NOT FLAG_INDEX EQUAL -1) MESSAGE(STATUS "Detected CUDA Compute Capability ${_CUDA_COMPUTE_CAPABILITY}") @@ -500,7 +566,7 @@ IF (KOKKOS_ENABLE_CUDA) SET(KOKKOS_ARCH_VOLTA ON) ENDIF() - IF (KOKKOS_ARCH_AMPERE80) + IF (KOKKOS_ARCH_AMPERE80 OR KOKKOS_ARCH_AMPERE86) SET(KOKKOS_ARCH_AMPERE ON) ENDIF() ENDIF() diff --git a/lib/kokkos/cmake/kokkos_compiler_id.cmake b/lib/kokkos/cmake/kokkos_compiler_id.cmake index e6600161f9..4434d6928f 100644 --- a/lib/kokkos/cmake/kokkos_compiler_id.cmake +++ b/lib/kokkos/cmake/kokkos_compiler_id.cmake @@ -27,6 +27,12 @@ IF(Kokkos_ENABLE_CUDA) PATHS ${PROJECT_SOURCE_DIR} PATH_SUFFIXES bin) + FIND_PROGRAM(Kokkos_NVCC_WRAPPER + NAMES nvcc_wrapper + HINTS ${PROJECT_SOURCE_DIR} + PATHS ${PROJECT_SOURCE_DIR} + PATH_SUFFIXES bin) + # check if compiler was set to nvcc_wrapper kokkos_internal_have_compiler_nvcc(${CMAKE_CXX_COMPILER}) # if launcher was found and nvcc_wrapper was not specified as @@ -37,7 +43,7 @@ IF(Kokkos_ENABLE_CUDA) # if the second argument matches the C++ compiler, it forwards the rest of the # args to nvcc_wrapper kokkos_internal_have_compiler_nvcc( - ${Kokkos_COMPILE_LAUNCHER} ${CMAKE_CXX_COMPILER} ${CMAKE_CXX_COMPILER} -DKOKKOS_DEPENDENCE) + ${Kokkos_COMPILE_LAUNCHER} ${Kokkos_NVCC_WRAPPER} ${CMAKE_CXX_COMPILER} ${CMAKE_CXX_COMPILER} -DKOKKOS_DEPENDENCE) SET(INTERNAL_USE_COMPILER_LAUNCHER true) ENDIF() ENDIF() @@ -55,32 +61,7 @@ IF(INTERNAL_HAVE_COMPILER_NVCC) SET(KOKKOS_CXX_COMPILER_VERSION ${TEMP_CXX_COMPILER_VERSION} CACHE STRING INTERNAL FORCE) MESSAGE(STATUS "Compiler Version: ${KOKKOS_CXX_COMPILER_VERSION}") IF(INTERNAL_USE_COMPILER_LAUNCHER) - IF(Kokkos_LAUNCH_COMPILER_INFO) - GET_FILENAME_COMPONENT(BASE_COMPILER_NAME ${CMAKE_CXX_COMPILER} NAME) - # does not have STATUS intentionally - MESSAGE("") - MESSAGE("Kokkos_LAUNCH_COMPILER_INFO (${Kokkos_COMPILE_LAUNCHER}):") - MESSAGE(" - Kokkos + CUDA backend requires the C++ files to be compiled as CUDA code.") - MESSAGE(" - kokkos_launch_compiler permits CMAKE_CXX_COMPILER to be set to a traditional C++ compiler when Kokkos_ENABLE_CUDA=ON") - MESSAGE(" by prefixing all the compile and link commands with the path to the script + CMAKE_CXX_COMPILER (${CMAKE_CXX_COMPILER}).") - MESSAGE(" - If any of the compile or link commands have CMAKE_CXX_COMPILER as the first argument, it replaces CMAKE_CXX_COMPILER with nvcc_wrapper.") - MESSAGE(" - If the compile or link command is not CMAKE_CXX_COMPILER, it just executes the command.") - MESSAGE(" - If using ccache, set CMAKE_CXX_COMPILER to nvcc_wrapper explicitly.") - MESSAGE(" - kokkos_compiler_launcher is available to downstream projects as well.") - MESSAGE(" - If CMAKE_CXX_COMPILER=nvcc_wrapper, all legacy behavior will be preserved during 'find_package(Kokkos)'") - MESSAGE(" - If CMAKE_CXX_COMPILER is not nvcc_wrapper, 'find_package(Kokkos)' will apply 'kokkos_compilation(GLOBAL)' unless separable compilation is enabled") - MESSAGE(" - This can be disabled via '-DKokkos_LAUNCH_COMPILER=OFF'") - MESSAGE(" - Use 'find_package(Kokkos COMPONENTS separable_compilation)' to enable separable compilation") - MESSAGE(" - Separable compilation allows you to control the scope of where the compiler transformation behavior (${BASE_COMPILER_NAME} -> nvcc_wrapper) is applied") - MESSAGE(" - The compiler transformation can be applied on a per-project, per-directory, per-target, and/or per-source-file basis") - MESSAGE(" - 'kokkos_compilation(PROJECT)' will apply the compiler transformation to all targets in a project/subproject") - MESSAGE(" - 'kokkos_compilation(TARGET [...])' will apply the compiler transformation to the specified target(s)") - MESSAGE(" - 'kokkos_compilation(SOURCE [...])' will apply the compiler transformation to the specified source file(s)") - MESSAGE(" - 'kokkos_compilation(DIRECTORY [...])' will apply the compiler transformation to the specified directories") - MESSAGE("") - ELSE() - MESSAGE(STATUS "kokkos_launch_compiler (${Kokkos_COMPILE_LAUNCHER}) is enabled... Set Kokkos_LAUNCH_COMPILER_INFO=ON for more info.") - ENDIF() + MESSAGE(STATUS "kokkos_launch_compiler (${Kokkos_COMPILE_LAUNCHER}) is enabled...") kokkos_compilation(GLOBAL) ENDIF() ENDIF() @@ -92,7 +73,11 @@ IF(Kokkos_ENABLE_HIP) OUTPUT_STRIP_TRAILING_WHITESPACE) STRING(REPLACE "\n" " - " INTERNAL_COMPILER_VERSION_ONE_LINE ${INTERNAL_COMPILER_VERSION} ) - SET(KOKKOS_CXX_COMPILER_ID HIP CACHE STRING INTERNAL FORCE) + + STRING(FIND ${INTERNAL_COMPILER_VERSION_ONE_LINE} "HIP version" INTERNAL_COMPILER_VERSION_CONTAINS_HIP) + IF(INTERNAL_COMPILER_VERSION_CONTAINS_HIP GREATER -1) + SET(KOKKOS_CXX_COMPILER_ID HIPCC CACHE STRING INTERNAL FORCE) + ENDIF() STRING(REGEX MATCH "[0-9]+\\.[0-9]+\\.[0-9]+" TEMP_CXX_COMPILER_VERSION ${INTERNAL_COMPILER_VERSION_ONE_LINE}) @@ -103,8 +88,7 @@ ENDIF() IF(KOKKOS_CXX_COMPILER_ID STREQUAL Clang) # The Cray compiler reports as Clang to most versions of CMake EXECUTE_PROCESS(COMMAND ${CMAKE_CXX_COMPILER} --version - COMMAND grep Cray - COMMAND wc -l + COMMAND grep -c Cray OUTPUT_VARIABLE INTERNAL_HAVE_CRAY_COMPILER OUTPUT_STRIP_TRAILING_WHITESPACE) IF (INTERNAL_HAVE_CRAY_COMPILER) #not actually Clang @@ -112,8 +96,7 @@ IF(KOKKOS_CXX_COMPILER_ID STREQUAL Clang) ENDIF() # The clang based Intel compiler reports as Clang to most versions of CMake EXECUTE_PROCESS(COMMAND ${CMAKE_CXX_COMPILER} --version - COMMAND grep icpx - COMMAND wc -l + COMMAND grep -c "DPC++\\|icpx" OUTPUT_VARIABLE INTERNAL_HAVE_INTEL_COMPILER OUTPUT_STRIP_TRAILING_WHITESPACE) IF (INTERNAL_HAVE_INTEL_COMPILER) #not actually Clang @@ -174,7 +157,7 @@ ELSEIF(KOKKOS_CXX_COMPILER_ID STREQUAL NVIDIA) MESSAGE(FATAL_ERROR "${KOKKOS_MESSAGE_TEXT}") ENDIF() SET(CMAKE_CXX_EXTENSIONS OFF CACHE BOOL "Kokkos turns off CXX extensions" FORCE) -ELSEIF(KOKKOS_CXX_COMPILER_ID STREQUAL HIP) +ELSEIF(KOKKOS_CXX_COMPILER_ID STREQUAL HIPCC) IF(KOKKOS_CXX_COMPILER_VERSION VERSION_LESS 3.8.0) MESSAGE(FATAL_ERROR "${KOKKOS_MESSAGE_TEXT}") ENDIF() diff --git a/lib/kokkos/cmake/kokkos_corner_cases.cmake b/lib/kokkos/cmake/kokkos_corner_cases.cmake index 3962c4b16e..a84ac2b630 100644 --- a/lib/kokkos/cmake/kokkos_corner_cases.cmake +++ b/lib/kokkos/cmake/kokkos_corner_cases.cmake @@ -49,11 +49,14 @@ ENDIF() IF (KOKKOS_CXX_STANDARD STREQUAL 17) IF (KOKKOS_CXX_COMPILER_ID STREQUAL GNU AND KOKKOS_CXX_COMPILER_VERSION VERSION_LESS 7) - MESSAGE(FATAL_ERROR "You have requested c++17 support for GCC ${KOKKOS_CXX_COMPILER_VERSION}. Although CMake has allowed this and GCC accepts -std=c++1z/c++17, GCC <= 6 does not properly support *this capture. Please reduce the C++ standard to 14 or upgrade the compiler if you do need C++17 support.") + MESSAGE(FATAL_ERROR "You have requested C++17 support for GCC ${KOKKOS_CXX_COMPILER_VERSION}. Although CMake has allowed this and GCC accepts -std=c++1z/c++17, GCC < 7 does not properly support *this capture. Please reduce the C++ standard to 14 or upgrade the compiler if you do need C++17 support.") ENDIF() IF (KOKKOS_CXX_COMPILER_ID STREQUAL NVIDIA AND KOKKOS_CXX_COMPILER_VERSION VERSION_LESS 11) - MESSAGE(FATAL_ERROR "You have requested c++17 support for NVCC ${KOKKOS_CXX_COMPILER_VERSION}. NVCC only supports C++17 from version 11 on. Please reduce the C++ standard to 14 or upgrade the compiler if you need C++17 support.") + MESSAGE(FATAL_ERROR "You have requested C++17 support for NVCC ${KOKKOS_CXX_COMPILER_VERSION}. NVCC only supports C++17 from version 11 on. Please reduce the C++ standard to 14 or upgrade the compiler if you need C++17 support.") + ENDIF() + IF (KOKKOS_CXX_COMPILER_ID STREQUAL NVIDIA AND KOKKOS_ENABLE_CUDA_CONSTEXPR) + MESSAGE(WARNING "You have requested -DKokkos_ENABLE_CUDA_CONSTEXPR=ON with C++17 support for NVCC ${KOKKOS_CXX_COMPILER_VERSION} which is known to trigger compiler bugs. See https://github.com/kokkos/kokkos/issues/3496") ENDIF() ENDIF() diff --git a/lib/kokkos/cmake/kokkos_enable_devices.cmake b/lib/kokkos/cmake/kokkos_enable_devices.cmake index 41ee10a8a0..445dad47ce 100644 --- a/lib/kokkos/cmake/kokkos_enable_devices.cmake +++ b/lib/kokkos/cmake/kokkos_enable_devices.cmake @@ -48,9 +48,6 @@ IF(KOKKOS_ENABLE_OPENMP) IF(KOKKOS_CLANG_IS_CRAY) SET(ClangOpenMPFlag -fopenmp) ENDIF() - IF(KOKKOS_CLANG_IS_INTEL) - SET(ClangOpenMPFlag -fiopenmp) - ENDIF() IF(KOKKOS_COMPILER_CLANG_MSVC) #for clang-cl expression /openmp yields an error, so directly add the specific Clang flag SET(ClangOpenMPFlag /clang:-fopenmp=libomp) @@ -64,6 +61,7 @@ IF(KOKKOS_ENABLE_OPENMP) COMPILER_SPECIFIC_FLAGS( COMPILER_ID KOKKOS_CXX_HOST_COMPILER_ID Clang -Xcompiler ${ClangOpenMPFlag} + IntelClang -Xcompiler -fiopenmp PGI -Xcompiler -mp Cray NO-VALUE-SPECIFIED XL -Xcompiler -qsmp=omp @@ -72,6 +70,7 @@ IF(KOKKOS_ENABLE_OPENMP) ELSE() COMPILER_SPECIFIC_FLAGS( Clang ${ClangOpenMPFlag} + IntelClang -fiopenmp AppleClang -Xpreprocessor -fopenmp PGI -mp Cray NO-VALUE-SPECIFIED @@ -152,3 +151,11 @@ IF (KOKKOS_ENABLE_HIP) ENDIF() KOKKOS_DEVICE_OPTION(SYCL OFF DEVICE "Whether to build SYCL backend") + +## SYCL has extra setup requirements, turn on Kokkos_Setup_SYCL.hpp in macros +IF (KOKKOS_ENABLE_SYCL) + IF(KOKKOS_CXX_STANDARD LESS 17) + MESSAGE(FATAL_ERROR "SYCL backend requires C++17 or newer!") + ENDIF() + LIST(APPEND DEVICE_SETUP_LIST SYCL) +ENDIF() diff --git a/lib/kokkos/cmake/kokkos_enable_options.cmake b/lib/kokkos/cmake/kokkos_enable_options.cmake index 5df498f373..95bce66c7b 100644 --- a/lib/kokkos/cmake/kokkos_enable_options.cmake +++ b/lib/kokkos/cmake/kokkos_enable_options.cmake @@ -48,6 +48,7 @@ KOKKOS_ENABLE_OPTION(COMPILER_WARNINGS OFF "Whether to print all compiler war KOKKOS_ENABLE_OPTION(PROFILING_LOAD_PRINT OFF "Whether to print information about which profiling tools got loaded") KOKKOS_ENABLE_OPTION(TUNING OFF "Whether to create bindings for tuning tools") KOKKOS_ENABLE_OPTION(AGGRESSIVE_VECTORIZATION OFF "Whether to aggressively vectorize loops") +KOKKOS_ENABLE_OPTION(LAUNCH_COMPILER ON "Whether to potentially use the launch compiler") IF (KOKKOS_ENABLE_CUDA) SET(KOKKOS_COMPILER_CUDA_VERSION "${KOKKOS_COMPILER_VERSION_MAJOR}${KOKKOS_COMPILER_VERSION_MINOR}") @@ -68,6 +69,15 @@ ELSE() ENDIF() KOKKOS_ENABLE_OPTION(COMPLEX_ALIGN ${COMPLEX_ALIGN_DEFAULT} "Whether to align Kokkos::complex to 2*alignof(RealType)") +IF (KOKKOS_ENABLE_TESTS) + SET(HEADER_SELF_CONTAINMENT_TESTS_DEFAULT ON) +ELSE() + SET(HEADER_SELF_CONTAINMENT_TESTS_DEFAULT OFF) +ENDIF() +KOKKOS_ENABLE_OPTION(HEADER_SELF_CONTAINMENT_TESTS ${HEADER_SELF_CONTAINMENT_TESTS_DEFAULT} "Enable header self-containment unit tests") +IF (NOT KOKKOS_ENABLE_TESTS AND KOKKOS_ENABLE_HEADER_SELF_CONTAINMENT_TESTS) + MESSAGE(WARNING "Kokkos_ENABLE_HEADER_SELF_CONTAINMENT_TESTS is ON but Kokkos_ENABLE_TESTS is OFF. Option will be ignored.") +ENDIF() IF (KOKKOS_ENABLE_CUDA AND (KOKKOS_CXX_COMPILER_ID STREQUAL Clang)) SET(CUDA_CONSTEXPR_DEFAULT ON) @@ -76,14 +86,14 @@ ELSE() ENDIF() KOKKOS_ENABLE_OPTION(CUDA_CONSTEXPR ${CUDA_CONSTEXPR_DEFAULT} "Whether to activate experimental relaxed constexpr functions") +Kokkos_ENABLE_OPTION(UNSUPPORTED_ARCHS OFF "Whether to allow architectures in backends Kokkos doesn't optimize for") + FUNCTION(check_device_specific_options) CMAKE_PARSE_ARGUMENTS(SOME "" "DEVICE" "OPTIONS" ${ARGN}) IF(NOT KOKKOS_ENABLE_${SOME_DEVICE}) FOREACH(OPTION ${SOME_OPTIONS}) - IF(CMAKE_VERSION VERSION_GREATER_EQUAL 3.14) - IF(NOT DEFINED CACHE{Kokkos_ENABLE_${OPTION}} OR NOT DEFINED CACHE{Kokkos_ENABLE_${SOME_DEVICE}}) - MESSAGE(FATAL_ERROR "Internal logic error: option '${OPTION}' or device '${SOME_DEVICE}' not recognized.") - ENDIF() + IF(NOT DEFINED CACHE{Kokkos_ENABLE_${OPTION}} OR NOT DEFINED CACHE{Kokkos_ENABLE_${SOME_DEVICE}}) + MESSAGE(FATAL_ERROR "Internal logic error: option '${OPTION}' or device '${SOME_DEVICE}' not recognized.") ENDIF() IF(KOKKOS_ENABLE_${OPTION}) MESSAGE(WARNING "Kokkos_ENABLE_${OPTION} is ON but ${SOME_DEVICE} backend is not enabled. Option will be ignored.") diff --git a/lib/kokkos/cmake/kokkos_functions.cmake b/lib/kokkos/cmake/kokkos_functions.cmake index 2b17d648b4..858322394d 100644 --- a/lib/kokkos/cmake/kokkos_functions.cmake +++ b/lib/kokkos/cmake/kokkos_functions.cmake @@ -169,9 +169,7 @@ MACRO(kokkos_export_imported_tpl NAME) ENDIF() SET(TPL_LINK_OPTIONS) - IF(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.13.0") - GET_TARGET_PROPERTY(TPL_LINK_OPTIONS ${NAME} INTERFACE_LINK_OPTIONS) - ENDIF() + GET_TARGET_PROPERTY(TPL_LINK_OPTIONS ${NAME} INTERFACE_LINK_OPTIONS) IF(TPL_LINK_OPTIONS) KOKKOS_APPEND_CONFIG_LINE("INTERFACE_LINK_OPTIONS ${TPL_LINK_OPTIONS}") ENDIF() @@ -230,9 +228,7 @@ MACRO(kokkos_import_tpl NAME) # I have still been getting errors about ROOT variables being ignored # I'm not sure if this is a scope issue - but make sure # the policy is set before we do any find_package calls - IF(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.12.0") - CMAKE_POLICY(SET CMP0074 NEW) - ENDIF() + CMAKE_POLICY(SET CMP0074 NEW) IF (KOKKOS_ENABLE_${NAME}) #Tack on a TPL here to make sure we avoid using anyone else's find @@ -314,7 +310,7 @@ MACRO(kokkos_create_imported_tpl NAME) CMAKE_PARSE_ARGUMENTS(TPL "INTERFACE" "LIBRARY" - "LINK_LIBRARIES;INCLUDES;COMPILE_OPTIONS;LINK_OPTIONS" + "LINK_LIBRARIES;INCLUDES;COMPILE_DEFINITIONS;COMPILE_OPTIONS;LINK_OPTIONS" ${ARGN}) @@ -334,6 +330,9 @@ MACRO(kokkos_create_imported_tpl NAME) IF(TPL_INCLUDES) TARGET_INCLUDE_DIRECTORIES(${NAME} INTERFACE ${TPL_INCLUDES}) ENDIF() + IF(TPL_COMPILE_DEFINITIONS) + TARGET_COMPILE_DEFINITIONS(${NAME} INTERFACE ${TPL_COMPILE_DEFINITIONS}) + ENDIF() IF(TPL_COMPILE_OPTIONS) TARGET_COMPILE_OPTIONS(${NAME} INTERFACE ${TPL_COMPILE_OPTIONS}) ENDIF() @@ -355,6 +354,10 @@ MACRO(kokkos_create_imported_tpl NAME) SET_TARGET_PROPERTIES(${NAME} PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${TPL_INCLUDES}") ENDIF() + IF(TPL_COMPILE_DEFINITIONS) + SET_TARGET_PROPERTIES(${NAME} PROPERTIES + INTERFACE_COMPILE_DEFINITIONS "${TPL_COMPILE_DEFINITIONS}") + ENDIF() IF(TPL_COMPILE_OPTIONS) SET_TARGET_PROPERTIES(${NAME} PROPERTIES INTERFACE_COMPILE_OPTIONS "${TPL_COMPILE_OPTIONS}") @@ -770,7 +773,7 @@ FUNCTION(kokkos_link_tpl TARGET) ENDFUNCTION() FUNCTION(COMPILER_SPECIFIC_OPTIONS_HELPER) - SET(COMPILERS NVIDIA PGI XL DEFAULT Cray Intel Clang AppleClang IntelClang GNU HIP Fujitsu) + SET(COMPILERS NVIDIA PGI XL DEFAULT Cray Intel Clang AppleClang IntelClang GNU HIPCC Fujitsu) CMAKE_PARSE_ARGUMENTS( PARSE "LINK_OPTIONS;COMPILE_OPTIONS;COMPILE_DEFINITIONS;LINK_LIBRARIES" @@ -926,6 +929,9 @@ ENDFUNCTION() # DIRECTORY --> all files in directory # PROJECT --> all files/targets in a project/subproject # +# NOTE: this is VERY DIFFERENT than the version in KokkosConfigCommon.cmake.in. +# This version explicitly uses nvcc_wrapper. +# FUNCTION(kokkos_compilation) # check whether the compiler already supports building CUDA KOKKOS_CXX_COMPILER_CUDA_TEST(Kokkos_CXX_COMPILER_COMPILES_CUDA) @@ -947,10 +953,21 @@ FUNCTION(kokkos_compilation) MESSAGE(FATAL_ERROR "Kokkos could not find 'kokkos_launch_compiler'. Please set '-DKokkos_COMPILE_LAUNCHER=/path/to/launcher'") ENDIF() + # find nvcc_wrapper + FIND_PROGRAM(Kokkos_NVCC_WRAPPER + NAMES nvcc_wrapper + HINTS ${PROJECT_SOURCE_DIR} + PATHS ${PROJECT_SOURCE_DIR} + PATH_SUFFIXES bin) + + IF(NOT Kokkos_COMPILE_LAUNCHER) + MESSAGE(FATAL_ERROR "Kokkos could not find 'nvcc_wrapper'. Please set '-DKokkos_COMPILE_LAUNCHER=/path/to/nvcc_wrapper'") + ENDIF() + IF(COMP_GLOBAL) # if global, don't bother setting others - SET_PROPERTY(GLOBAL PROPERTY RULE_LAUNCH_COMPILE "${Kokkos_COMPILE_LAUNCHER} ${CMAKE_CXX_COMPILER}") - SET_PROPERTY(GLOBAL PROPERTY RULE_LAUNCH_LINK "${Kokkos_COMPILE_LAUNCHER} ${CMAKE_CXX_COMPILER}") + SET_PROPERTY(GLOBAL PROPERTY RULE_LAUNCH_COMPILE "${Kokkos_COMPILE_LAUNCHER} ${Kokkos_NVCC_WRAPPER} ${CMAKE_CXX_COMPILER}") + SET_PROPERTY(GLOBAL PROPERTY RULE_LAUNCH_LINK "${Kokkos_COMPILE_LAUNCHER} ${Kokkos_NVCC_WRAPPER} ${CMAKE_CXX_COMPILER}") ELSE() FOREACH(_TYPE PROJECT DIRECTORY TARGET SOURCE) # make project/subproject scoping easy, e.g. KokkosCompilation(PROJECT) after project(...) @@ -961,8 +978,8 @@ FUNCTION(kokkos_compilation) # set the properties if defined IF(COMP_${_TYPE}) # MESSAGE(STATUS "Using nvcc_wrapper :: ${_TYPE} :: ${COMP_${_TYPE}}") - SET_PROPERTY(${_TYPE} ${COMP_${_TYPE}} PROPERTY RULE_LAUNCH_COMPILE "${Kokkos_COMPILE_LAUNCHER} ${CMAKE_CXX_COMPILER}") - SET_PROPERTY(${_TYPE} ${COMP_${_TYPE}} PROPERTY RULE_LAUNCH_LINK "${Kokkos_COMPILE_LAUNCHER} ${CMAKE_CXX_COMPILER}") + SET_PROPERTY(${_TYPE} ${COMP_${_TYPE}} PROPERTY RULE_LAUNCH_COMPILE "${Kokkos_COMPILE_LAUNCHER} ${Kokkos_NVCC_WRAPPER} ${CMAKE_CXX_COMPILER}") + SET_PROPERTY(${_TYPE} ${COMP_${_TYPE}} PROPERTY RULE_LAUNCH_LINK "${Kokkos_COMPILE_LAUNCHER} ${Kokkos_NVCC_WRAPPER} ${CMAKE_CXX_COMPILER}") ENDIF() ENDFOREACH() ENDIF() diff --git a/lib/kokkos/cmake/kokkos_test_cxx_std.cmake b/lib/kokkos/cmake/kokkos_test_cxx_std.cmake index 1d7da922eb..707fb000af 100644 --- a/lib/kokkos/cmake/kokkos_test_cxx_std.cmake +++ b/lib/kokkos/cmake/kokkos_test_cxx_std.cmake @@ -86,6 +86,19 @@ ELSE() MESSAGE(FATAL_ERROR "Unknown C++ standard ${KOKKOS_CXX_STANDARD} - must be 14, 17, or 20") ENDIF() +# Enforce that we can compile a simple C++14 program + +TRY_COMPILE(CAN_COMPILE_CPP14 + ${KOKKOS_TOP_BUILD_DIR}/corner_cases + ${KOKKOS_SOURCE_DIR}/cmake/compile_tests/cplusplus14.cpp + OUTPUT_VARIABLE ERROR_MESSAGE + CXX_STANDARD 14 +) +if (NOT CAN_COMPILE_CPP14) + UNSET(CAN_COMPILE_CPP14 CACHE) #make sure CMake always re-runs this + MESSAGE(FATAL_ERROR "C++${KOKKOS_CXX_STANDARD}-compliant compiler detected, but unable to compile C++14 or later program. Verify that ${CMAKE_CXX_COMPILER_ID}:${CMAKE_CXX_COMPILER_VERSION} is set up correctly (e.g., check that correct library headers are being used).\nFailing output:\n ${ERROR_MESSAGE}") +ENDIF() +UNSET(CAN_COMPILE_CPP14 CACHE) #make sure CMake always re-runs this # Enforce that extensions are turned off for nvcc_wrapper. diff --git a/lib/kokkos/cmake/kokkos_tpls.cmake b/lib/kokkos/cmake/kokkos_tpls.cmake index b58d3696ea..d8d044c9d7 100644 --- a/lib/kokkos/cmake/kokkos_tpls.cmake +++ b/lib/kokkos/cmake/kokkos_tpls.cmake @@ -1,5 +1,6 @@ KOKKOS_CFG_DEPENDS(TPLS OPTIONS) KOKKOS_CFG_DEPENDS(TPLS DEVICES) +KOKKOS_CFG_DEPENDS(TPLS COMPILER_ID) FUNCTION(KOKKOS_TPL_OPTION PKG DEFAULT) CMAKE_PARSE_ARGUMENTS(PARSED @@ -38,6 +39,12 @@ IF(KOKKOS_ENABLE_MEMKIND) ENDIF() KOKKOS_TPL_OPTION(CUDA ${Kokkos_ENABLE_CUDA} TRIBITS CUDA) KOKKOS_TPL_OPTION(LIBRT Off) +IF(KOKKOS_ENABLE_HIP AND NOT KOKKOS_CXX_COMPILER_ID STREQUAL HIPCC) + SET(ROCM_DEFAULT ON) +ELSE() + SET(ROCM_DEFAULT OFF) +ENDIF() +KOKKOS_TPL_OPTION(ROCM ${ROCM_DEFAULT}) IF (WIN32) SET(LIBDL_DEFAULT Off) @@ -70,6 +77,7 @@ KOKKOS_IMPORT_TPL(LIBRT) KOKKOS_IMPORT_TPL(LIBDL) KOKKOS_IMPORT_TPL(MEMKIND) KOKKOS_IMPORT_TPL(PTHREAD INTERFACE) +KOKKOS_IMPORT_TPL(ROCM INTERFACE) #Convert list to newlines (which CMake doesn't always like in cache variables) STRING(REPLACE ";" "\n" KOKKOS_TPL_EXPORT_TEMP "${KOKKOS_TPL_EXPORTS}") diff --git a/lib/kokkos/cmake/kokkos_tribits.cmake b/lib/kokkos/cmake/kokkos_tribits.cmake index 059fb192f0..afa036066a 100644 --- a/lib/kokkos/cmake/kokkos_tribits.cmake +++ b/lib/kokkos/cmake/kokkos_tribits.cmake @@ -141,39 +141,54 @@ FUNCTION(KOKKOS_ADD_EXECUTABLE ROOT_NAME) ENDFUNCTION() FUNCTION(KOKKOS_ADD_EXECUTABLE_AND_TEST ROOT_NAME) -CMAKE_PARSE_ARGUMENTS(PARSE - "" - "" - "SOURCES;CATEGORIES;ARGS" - ${ARGN}) -VERIFY_EMPTY(KOKKOS_ADD_EXECUTABLE_AND_TEST ${PARSE_UNPARSED_ARGUMENTS}) + CMAKE_PARSE_ARGUMENTS(PARSE + "" + "" + "SOURCES;CATEGORIES;ARGS" + ${ARGN}) + VERIFY_EMPTY(KOKKOS_ADD_EXECUTABLE_AND_TEST ${PARSE_UNPARSED_ARGUMENTS}) -IF (KOKKOS_HAS_TRILINOS) - IF(DEFINED PARSE_ARGS) - STRING(REPLACE ";" " " PARSE_ARGS "${PARSE_ARGS}") - ENDIF() - TRIBITS_ADD_EXECUTABLE_AND_TEST( - ${ROOT_NAME} - SOURCES ${PARSE_SOURCES} - TESTONLYLIBS kokkos_gtest - NUM_MPI_PROCS 1 - COMM serial mpi - ARGS ${PARSE_ARGS} - CATEGORIES ${PARSE_CATEGORIES} - SOURCES ${PARSE_SOURCES} - FAIL_REGULAR_EXPRESSION " FAILED " - ARGS ${PARSE_ARGS} - ) -ELSE() - KOKKOS_ADD_TEST_EXECUTABLE(${ROOT_NAME} - SOURCES ${PARSE_SOURCES} - ) - KOKKOS_ADD_TEST(NAME ${ROOT_NAME} - EXE ${ROOT_NAME} - FAIL_REGULAR_EXPRESSION " FAILED " - ARGS ${PARSE_ARGS} - ) -ENDIF() + IF (KOKKOS_HAS_TRILINOS) + IF(DEFINED PARSE_ARGS) + STRING(REPLACE ";" " " PARSE_ARGS "${PARSE_ARGS}") + ENDIF() + TRIBITS_ADD_EXECUTABLE_AND_TEST( + ${ROOT_NAME} + SOURCES ${PARSE_SOURCES} + TESTONLYLIBS kokkos_gtest + NUM_MPI_PROCS 1 + COMM serial mpi + ARGS ${PARSE_ARGS} + CATEGORIES ${PARSE_CATEGORIES} + SOURCES ${PARSE_SOURCES} + FAIL_REGULAR_EXPRESSION " FAILED " + ARGS ${PARSE_ARGS} + ) + ELSE() + KOKKOS_ADD_TEST_EXECUTABLE(${ROOT_NAME} + SOURCES ${PARSE_SOURCES} + ) + IF (PARSE_ARGS) + SET(TEST_NUMBER 0) + FOREACH (ARG_STR ${PARSE_ARGS}) + # This is passed as a single string blob to match TriBITS behavior + # We need this to be turned into a list + STRING(REPLACE " " ";" ARG_STR_LIST ${ARG_STR}) + LIST(APPEND TEST_NAME "${ROOT_NAME}${TEST_NUMBER}") + MATH(EXPR TEST_NUMBER "${TEST_NUMBER} + 1") + KOKKOS_ADD_TEST(NAME ${TEST_NAME} + EXE ${ROOT_NAME} + FAIL_REGULAR_EXPRESSION " FAILED " + ARGS ${ARG_STR_LIST} + ) + ENDFOREACH() + ELSE() + KOKKOS_ADD_TEST(NAME ${ROOT_NAME} + EXE ${ROOT_NAME} + FAIL_REGULAR_EXPRESSION " FAILED " + ) + ENDIF() + ENDIF() ENDFUNCTION() FUNCTION(KOKKOS_SET_EXE_PROPERTY ROOT_NAME) @@ -301,11 +316,26 @@ ENDMACRO() ## Includes generated header files, scripts such as nvcc_wrapper and hpcbind, ## as well as other files provided through plugins. MACRO(KOKKOS_INSTALL_ADDITIONAL_FILES) - # kokkos_launch_compiler is used by Kokkos to prefix compiler commands so that they forward to nvcc_wrapper + + # kokkos_launch_compiler is used by Kokkos to prefix compiler commands so that they forward to original kokkos compiler + # if nvcc_wrapper was not used as CMAKE_CXX_COMPILER, configure the original compiler into kokkos_launch_compiler + IF(NOT "${CMAKE_CXX_COMPILER}" MATCHES "nvcc_wrapper") + SET(NVCC_WRAPPER_DEFAULT_COMPILER "${CMAKE_CXX_COMPILER}") + ELSE() + IF(NOT "$ENV{NVCC_WRAPPER_DEFAULT_COMPILER}" STREQUAL "") + SET(NVCC_WRAPPER_DEFAULT_COMPILER "$ENV{NVCC_WRAPPER_DEFAULT_COMPILER}") + ENDIF() + ENDIF() + + CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/bin/kokkos_launch_compiler + ${PROJECT_BINARY_DIR}/temp/kokkos_launch_compiler + @ONLY) + INSTALL(PROGRAMS "${CMAKE_CURRENT_SOURCE_DIR}/bin/nvcc_wrapper" "${CMAKE_CURRENT_SOURCE_DIR}/bin/hpcbind" "${CMAKE_CURRENT_SOURCE_DIR}/bin/kokkos_launch_compiler" + "${PROJECT_BINARY_DIR}/temp/kokkos_launch_compiler" DESTINATION ${CMAKE_INSTALL_BINDIR}) INSTALL(FILES "${CMAKE_CURRENT_BINARY_DIR}/KokkosCore_config.h" @@ -313,7 +343,7 @@ MACRO(KOKKOS_INSTALL_ADDITIONAL_FILES) "${CMAKE_CURRENT_BINARY_DIR}/KokkosCore_Config_SetupBackend.hpp" "${CMAKE_CURRENT_BINARY_DIR}/KokkosCore_Config_DeclareBackend.hpp" "${CMAKE_CURRENT_BINARY_DIR}/KokkosCore_Config_PostInclude.hpp" - DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) + DESTINATION ${KOKKOS_HEADER_DIR}) ENDMACRO() FUNCTION(KOKKOS_SET_LIBRARY_PROPERTIES LIBRARY_NAME) @@ -330,24 +360,12 @@ FUNCTION(KOKKOS_SET_LIBRARY_PROPERTIES LIBRARY_NAME) ${LIBRARY_NAME} PUBLIC $<$:${KOKKOS_LINK_OPTIONS}> ) - ELSEIF(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.13") + ELSE() #I can use link options #just assume CXX linkage TARGET_LINK_OPTIONS( ${LIBRARY_NAME} PUBLIC ${KOKKOS_LINK_OPTIONS} ) - ELSE() - #assume CXX linkage, we have no good way to check otherwise - IF (PARSE_PLAIN_STYLE) - TARGET_LINK_LIBRARIES( - ${LIBRARY_NAME} ${KOKKOS_LINK_OPTIONS} - ) - ELSE() - #well, have to do it the wrong way for now - TARGET_LINK_LIBRARIES( - ${LIBRARY_NAME} PUBLIC ${KOKKOS_LINK_OPTIONS} - ) - ENDIF() ENDIF() TARGET_COMPILE_OPTIONS( @@ -448,6 +466,13 @@ FUNCTION(KOKKOS_INTERNAL_ADD_LIBRARY LIBRARY_NAME) ${PARSE_SOURCES} ) + IF(PARSE_SHARED OR BUILD_SHARED_LIBS) + SET_TARGET_PROPERTIES(${LIBRARY_NAME} PROPERTIES + VERSION ${Kokkos_VERSION} + SOVERSION ${Kokkos_VERSION_MAJOR}.${Kokkos_VERSION_MINOR} + ) + ENDIF() + KOKKOS_INTERNAL_ADD_LIBRARY_INSTALL(${LIBRARY_NAME}) #In case we are building in-tree, add an alias name diff --git a/lib/kokkos/containers/src/CMakeLists.txt b/lib/kokkos/containers/src/CMakeLists.txt index 7000624b6b..98655896d4 100644 --- a/lib/kokkos/containers/src/CMakeLists.txt +++ b/lib/kokkos/containers/src/CMakeLists.txt @@ -26,8 +26,6 @@ KOKKOS_ADD_LIBRARY( HEADERS ${KOKKOS_CONTAINER_HEADERS} ) -SET_TARGET_PROPERTIES(kokkoscontainers PROPERTIES VERSION ${Kokkos_VERSION}) - KOKKOS_LIB_INCLUDE_DIRECTORIES(kokkoscontainers ${KOKKOS_TOP_BUILD_DIR} ${CMAKE_CURRENT_BINARY_DIR} @@ -36,4 +34,3 @@ KOKKOS_LIB_INCLUDE_DIRECTORIES(kokkoscontainers KOKKOS_LINK_INTERNAL_LIBRARY(kokkoscontainers kokkoscore) #----------------------------------------------------------------------------- - diff --git a/lib/kokkos/containers/src/Kokkos_DualView.hpp b/lib/kokkos/containers/src/Kokkos_DualView.hpp index 689f0eb2ed..45710d1f73 100644 --- a/lib/kokkos/containers/src/Kokkos_DualView.hpp +++ b/lib/kokkos/containers/src/Kokkos_DualView.hpp @@ -91,6 +91,25 @@ namespace Kokkos { * behavior. Please see the documentation of Kokkos::View for * examples. The default suffices for most users. */ + +namespace Impl { + +#ifdef KOKKOS_ENABLE_CUDA + +inline const Kokkos::Cuda& get_cuda_space(const Kokkos::Cuda& in) { return in; } + +inline const Kokkos::Cuda& get_cuda_space() { + return *Kokkos::Impl::cuda_get_deep_copy_space(); +} + +template +inline const Kokkos::Cuda& get_cuda_space(const NonCudaExecSpace&) { + return get_cuda_space(); +} + +#endif // KOKKOS_ENABLE_CUDA + +} // namespace Impl template class DualView : public ViewTraits { @@ -295,6 +314,53 @@ class DualView : public ViewTraits { "DualView constructed with incompatible views"); } } + // does the DualView have only one device + struct impl_dualview_is_single_device { + enum : bool { + value = std::is_same::value + }; + }; + + // does the given device match the device of t_dev? + template + struct impl_device_matches_tdev_device { + enum : bool { + value = std::is_same::value + }; + }; + // does the given device match the device of t_host? + template + struct impl_device_matches_thost_device { + enum : bool { + value = std::is_same::value + }; + }; + + // does the given device match the execution space of t_host? + template + struct impl_device_matches_thost_exec { + enum : bool { + value = std::is_same::value + }; + }; + + // does the given device match the execution space of t_dev? + template + struct impl_device_matches_tdev_exec { + enum : bool { + value = std::is_same::value + }; + }; + + // does the given device's memory space match the memory space of t_dev? + template + struct impl_device_matches_tdev_memory_space { + enum : bool { + value = std::is_same::value + }; + }; //@} //! \name Methods for synchronizing, marking as modified, and getting Views. @@ -302,7 +368,7 @@ class DualView : public ViewTraits { /// \brief Return a View on a specific device \c Device. /// - /// Please don't be afraid of the if_c expression in the return + /// Please don't be afraid of the nested if_c expressions in the return /// value's type. That just tells the method what the return type /// should be: t_dev if the \c Device template parameter matches /// this DualView's device type, else t_host. @@ -323,10 +389,17 @@ class DualView : public ViewTraits { /// typename dual_view_type::t_host hostView = DV.view (); /// \endcode template - KOKKOS_INLINE_FUNCTION const typename Impl::if_c< - std::is_same::value, - t_dev, t_host>::type& + KOKKOS_INLINE_FUNCTION const typename std::conditional_t< + impl_device_matches_tdev_device::value, t_dev, + typename std::conditional_t< + impl_device_matches_thost_device::value, t_host, + typename std::conditional_t< + impl_device_matches_thost_exec::value, t_host, + typename std::conditional_t< + impl_device_matches_tdev_exec::value, t_dev, + typename std::conditional_t< + impl_device_matches_tdev_memory_space::value, + t_dev, t_host> > > > > view() const { constexpr bool device_is_memspace = std::is_same::value; @@ -463,6 +536,7 @@ class DualView : public ViewTraits { true); } } + /// \brief Update data on device or host only if data in the other /// space has been marked as modified. /// @@ -480,12 +554,9 @@ class DualView : public ViewTraits { /// the data in either View. You must manually mark modified data /// as modified, by calling the modify() method with the /// appropriate template parameter. - template - void sync(const typename std::enable_if< - (std::is_same::value) || - (std::is_same::value), - int>::type& = 0) { + // deliberately passing args by cref as they're used multiple times + template + void sync_impl(std::true_type, Args const&... args) { if (modified_flags.data() == nullptr) return; int dev = get_device_side(); @@ -497,12 +568,12 @@ class DualView : public ViewTraits { Kokkos::CudaUVMSpace>::value) { if (d_view.data() == h_view.data()) Kokkos::Impl::cuda_prefetch_pointer( - Kokkos::Cuda(), d_view.data(), + Impl::get_cuda_space(args...), d_view.data(), sizeof(typename t_dev::value_type) * d_view.span(), true); } #endif - deep_copy(d_view, h_view); + deep_copy(args..., d_view, h_view); modified_flags(0) = modified_flags(1) = 0; impl_report_device_sync(); } @@ -514,12 +585,12 @@ class DualView : public ViewTraits { Kokkos::CudaUVMSpace>::value) { if (d_view.data() == h_view.data()) Kokkos::Impl::cuda_prefetch_pointer( - Kokkos::Cuda(), d_view.data(), + Impl::get_cuda_space(args...), d_view.data(), sizeof(typename t_dev::value_type) * d_view.span(), false); } #endif - deep_copy(h_view, d_view); + deep_copy(args..., h_view, d_view); modified_flags(0) = modified_flags(1) = 0; impl_report_host_sync(); } @@ -533,10 +604,26 @@ class DualView : public ViewTraits { template void sync(const typename std::enable_if< - (!std::is_same::value) || + (std::is_same::value) || (std::is_same::value), int>::type& = 0) { + sync_impl(std::true_type{}); + } + + template + void sync(const ExecutionSpace& exec, + const typename std::enable_if< + (std::is_same::value) || + (std::is_same::value), + int>::type& = 0) { + sync_impl(std::true_type{}, exec); + } + + // deliberately passing args by cref as they're used multiple times + template + void sync_impl(std::false_type, Args const&...) { if (modified_flags.data() == nullptr) return; int dev = get_device_side(); @@ -557,7 +644,27 @@ class DualView : public ViewTraits { } } - void sync_host() { + template + void sync(const typename std::enable_if< + (!std::is_same::value) || + (std::is_same::value), + int>::type& = 0) { + sync_impl(std::false_type{}); + } + template + void sync(const ExecutionSpace& exec, + const typename std::enable_if< + (!std::is_same::value) || + (std::is_same::value), + int>::type& = 0) { + sync_impl(std::false_type{}, exec); + } + + // deliberately passing args by cref as they're used multiple times + template + void sync_host_impl(Args const&... args) { if (!std::is_same::value) Impl::throw_runtime_exception( @@ -569,18 +676,26 @@ class DualView : public ViewTraits { Kokkos::CudaUVMSpace>::value) { if (d_view.data() == h_view.data()) Kokkos::Impl::cuda_prefetch_pointer( - Kokkos::Cuda(), d_view.data(), + Impl::get_cuda_space(args...), d_view.data(), sizeof(typename t_dev::value_type) * d_view.span(), false); } #endif - deep_copy(h_view, d_view); + deep_copy(args..., h_view, d_view); modified_flags(1) = modified_flags(0) = 0; impl_report_host_sync(); } } - void sync_device() { + template + void sync_host(const ExecSpace& exec) { + sync_host_impl(exec); + } + void sync_host() { sync_host_impl(); } + + // deliberately passing args by cref as they're used multiple times + template + void sync_device_impl(Args const&... args) { if (!std::is_same::value) Impl::throw_runtime_exception( @@ -592,17 +707,23 @@ class DualView : public ViewTraits { Kokkos::CudaUVMSpace>::value) { if (d_view.data() == h_view.data()) Kokkos::Impl::cuda_prefetch_pointer( - Kokkos::Cuda(), d_view.data(), + Impl::get_cuda_space(args...), d_view.data(), sizeof(typename t_dev::value_type) * d_view.span(), true); } #endif - deep_copy(d_view, h_view); + deep_copy(args..., d_view, h_view); modified_flags(1) = modified_flags(0) = 0; impl_report_device_sync(); } } + template + void sync_device(const ExecSpace& exec) { + sync_device_impl(exec); + } + void sync_device() { sync_device_impl(); } + template bool need_sync() const { if (modified_flags.data() == nullptr) return false; @@ -658,6 +779,7 @@ class DualView : public ViewTraits { template void modify() { if (modified_flags.data() == nullptr) return; + if (impl_dualview_is_single_device::value) return; int dev = get_device_side(); if (dev == 1) { // if Device is the same as DualView's device type @@ -690,6 +812,7 @@ class DualView : public ViewTraits { } inline void modify_host() { + if (impl_dualview_is_single_device::value) return; if (modified_flags.data() != nullptr) { modified_flags(0) = (modified_flags(1) > modified_flags(0) ? modified_flags(1) @@ -710,6 +833,7 @@ class DualView : public ViewTraits { } inline void modify_device() { + if (impl_dualview_is_single_device::value) return; if (modified_flags.data() != nullptr) { modified_flags(1) = (modified_flags(1) > modified_flags(0) ? modified_flags(1) diff --git a/lib/kokkos/containers/src/Kokkos_DynRankView.hpp b/lib/kokkos/containers/src/Kokkos_DynRankView.hpp index c66d7a5f36..c6323fef93 100644 --- a/lib/kokkos/containers/src/Kokkos_DynRankView.hpp +++ b/lib/kokkos/containers/src/Kokkos_DynRankView.hpp @@ -245,13 +245,10 @@ KOKKOS_INLINE_FUNCTION bool dyn_rank_view_verify_operator_bounds( return (size_t(i) < map.extent(R)) && dyn_rank_view_verify_operator_bounds(rank, map, args...); } else if (i != 0) { - // FIXME_SYCL SYCL doesn't allow printf in kernels -#ifndef KOKKOS_ENABLE_SYCL - printf( + KOKKOS_IMPL_DO_NOT_USE_PRINTF( "DynRankView Debug Bounds Checking Error: at rank %u\n Extra " "arguments beyond the rank must be zero \n", R); -#endif return (false) && dyn_rank_view_verify_operator_bounds(rank, map, args...); } else { @@ -575,37 +572,22 @@ class DynRankView : public ViewTraits { (is_layout_left || is_layout_right || is_layout_stride) }; - template ::accessible> - struct verify_space { - KOKKOS_FORCEINLINE_FUNCTION static void check() {} - }; - - template - struct verify_space { - KOKKOS_FORCEINLINE_FUNCTION static void check() { - Kokkos::abort( - "Kokkos::DynRankView ERROR: attempt to access inaccessible memory " - "space"); - }; - }; - // Bounds checking macros #if defined(KOKKOS_ENABLE_DEBUG_BOUNDS_CHECK) // rank of the calling operator - included as first argument in ARG -#define KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(ARG) \ - DynRankView::template verify_space< \ - Kokkos::Impl::ActiveExecutionMemorySpace>::check(); \ - Kokkos::Impl::dyn_rank_view_verify_operator_bounds< \ - typename traits::memory_space> \ +#define KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(ARG) \ + Kokkos::Impl::verify_space::check(); \ + Kokkos::Impl::dyn_rank_view_verify_operator_bounds< \ + typename traits::memory_space> \ ARG; #else -#define KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(ARG) \ - DynRankView::template verify_space< \ - Kokkos::Impl::ActiveExecutionMemorySpace>::check(); +#define KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(ARG) \ + Kokkos::Impl::verify_space::check(); #endif diff --git a/lib/kokkos/containers/src/Kokkos_DynamicView.hpp b/lib/kokkos/containers/src/Kokkos_DynamicView.hpp index 06bd556661..cc949d4c55 100644 --- a/lib/kokkos/containers/src/Kokkos_DynamicView.hpp +++ b/lib/kokkos/containers/src/Kokkos_DynamicView.hpp @@ -76,6 +76,12 @@ struct ChunkArraySpace { using memory_space = typename Kokkos::Experimental::HIPHostPinnedSpace; }; #endif +#ifdef KOKKOS_ENABLE_SYCL +template <> +struct ChunkArraySpace { + using memory_space = typename Kokkos::Experimental::SYCLSharedUSMSpace; +}; +#endif } // end namespace Impl /** \brief Dynamic views are restricted to rank-one and no layout. diff --git a/lib/kokkos/containers/src/Kokkos_OffsetView.hpp b/lib/kokkos/containers/src/Kokkos_OffsetView.hpp index 4fd084338e..0f21a08ba3 100644 --- a/lib/kokkos/containers/src/Kokkos_OffsetView.hpp +++ b/lib/kokkos/containers/src/Kokkos_OffsetView.hpp @@ -377,34 +377,20 @@ class OffsetView : public ViewTraits { std::is_same::value && (is_layout_left || is_layout_right || is_layout_stride); - template ::accessible> - struct verify_space { - KOKKOS_FORCEINLINE_FUNCTION static void check() {} - }; - - template - struct verify_space { - KOKKOS_FORCEINLINE_FUNCTION static void check() { - Kokkos::abort( - "Kokkos::View ERROR: attempt to access inaccessible memory space"); - }; - }; - #if defined(KOKKOS_ENABLE_DEBUG_BOUNDS_CHECK) -#define KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY(ARG) \ - OffsetView::template verify_space< \ - Kokkos::Impl::ActiveExecutionMemorySpace>::check(); \ - Kokkos::Experimental::Impl::offsetview_verify_operator_bounds< \ - typename traits::memory_space> \ +#define KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY(ARG) \ + Kokkos::Impl::verify_space::check(); \ + Kokkos::Experimental::Impl::offsetview_verify_operator_bounds< \ + typename traits::memory_space> \ ARG; #else -#define KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY(ARG) \ - OffsetView::template verify_space< \ - Kokkos::Impl::ActiveExecutionMemorySpace>::check(); +#define KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY(ARG) \ + Kokkos::Impl::verify_space::check(); #endif public: diff --git a/lib/kokkos/containers/src/Kokkos_ScatterView.hpp b/lib/kokkos/containers/src/Kokkos_ScatterView.hpp index 5e18f5a80e..dcd4cf73e5 100644 --- a/lib/kokkos/containers/src/Kokkos_ScatterView.hpp +++ b/lib/kokkos/containers/src/Kokkos_ScatterView.hpp @@ -649,13 +649,13 @@ struct ReduceDuplicatesBase { size_t stride; size_t start; size_t n; - ReduceDuplicatesBase(ValueType const* src_in, ValueType* dest_in, - size_t stride_in, size_t start_in, size_t n_in, - std::string const& name) + ReduceDuplicatesBase(ExecSpace const& exec_space, ValueType const* src_in, + ValueType* dest_in, size_t stride_in, size_t start_in, + size_t n_in, std::string const& name) : src(src_in), dst(dest_in), stride(stride_in), start(start_in), n(n_in) { parallel_for( std::string("Kokkos::ScatterView::ReduceDuplicates [") + name + "]", - RangePolicy(0, stride), + RangePolicy(exec_space, 0, stride), static_cast(*this)); } }; @@ -667,9 +667,10 @@ template struct ReduceDuplicates : public ReduceDuplicatesBase { using Base = ReduceDuplicatesBase; - ReduceDuplicates(ValueType const* src_in, ValueType* dst_in, size_t stride_in, - size_t start_in, size_t n_in, std::string const& name) - : Base(src_in, dst_in, stride_in, start_in, n_in, name) {} + ReduceDuplicates(ExecSpace const& exec_space, ValueType const* src_in, + ValueType* dst_in, size_t stride_in, size_t start_in, + size_t n_in, std::string const& name) + : Base(exec_space, src_in, dst_in, stride_in, start_in, n_in, name) {} KOKKOS_FORCEINLINE_FUNCTION void operator()(size_t i) const { for (size_t j = Base::start; j < Base::n; ++j) { ScatterValue struct ResetDuplicatesBase { using Derived = ResetDuplicates; ValueType* data; - ResetDuplicatesBase(ValueType* data_in, size_t size_in, - std::string const& name) + ResetDuplicatesBase(ExecSpace const& exec_space, ValueType* data_in, + size_t size_in, std::string const& name) : data(data_in) { parallel_for( std::string("Kokkos::ScatterView::ResetDuplicates [") + name + "]", - RangePolicy(0, size_in), + RangePolicy(exec_space, 0, size_in), static_cast(*this)); } }; @@ -703,8 +704,9 @@ struct ResetDuplicatesBase { template struct ResetDuplicates : public ResetDuplicatesBase { using Base = ResetDuplicatesBase; - ResetDuplicates(ValueType* data_in, size_t size_in, std::string const& name) - : Base(data_in, size_in, name) {} + ResetDuplicates(ExecSpace const& exec_space, ValueType* data_in, + size_t size_in, std::string const& name) + : Base(exec_space, data_in, size_in, name) {} KOKKOS_FORCEINLINE_FUNCTION void operator()(size_t i) const { ScatterValue @@ -713,6 +715,16 @@ struct ResetDuplicates : public ResetDuplicatesBase { } }; +template +void check_scatter_view_allocation_properties_argument( + ViewCtorProp const&) { + static_assert(ViewCtorProp::has_execution_space && + ViewCtorProp::has_label && + ViewCtorProp::initialize, + "Allocation property must have an execution name as well as a " + "label, and must perform the view initialization"); +} + } // namespace Experimental } // namespace Impl } // namespace Kokkos @@ -762,10 +774,26 @@ class ScatterView const& original_view) : internal_view(original_view) {} + template + ScatterView(execution_space const& /* exec_space */, + View const& original_view) + : internal_view(original_view) {} + template ScatterView(std::string const& name, Dims... dims) : internal_view(name, dims...) {} + // This overload allows specifying an execution space instance to be + // used by passing, e.g., Kokkos::view_alloc(exec_space, "label") as + // first argument. + template + ScatterView(::Kokkos::Impl::ViewCtorProp const& arg_prop, Dims... dims) + : internal_view(arg_prop, dims...) { + using ::Kokkos::Impl::Experimental:: + check_scatter_view_allocation_properties_argument; + check_scatter_view_allocation_properties_argument(arg_prop); + } + template KOKKOS_FUNCTION ScatterView( const ScatterView void contribute_into(View const& dest) const { + contribute_into(execution_space(), dest); + } + + template + void contribute_into(execution_space const& exec_space, + View const& dest) const { using dest_type = View; static_assert(std::is_same::value, "ScatterView contribute destination has different layout"); static_assert( - Kokkos::Impl::VerifyExecutionCanAccessMemorySpace< - memory_space, typename dest_type::memory_space>::value, + Kokkos::Impl::SpaceAccessibility< + execution_space, typename dest_type::memory_space>::accessible, "ScatterView contribute destination memory space not accessible"); if (dest.data() == internal_view.data()) return; Kokkos::Impl::Experimental::ReduceDuplicates( - internal_view.data(), dest.data(), 0, 0, 1, internal_view.label()); + exec_space, internal_view.data(), dest.data(), 0, 0, 1, + internal_view.label()); } - void reset() { + void reset(execution_space const& exec_space = execution_space()) { Kokkos::Impl::Experimental::ResetDuplicates( - internal_view.data(), internal_view.size(), internal_view.label()); + exec_space, internal_view.data(), internal_view.size(), + internal_view.label()); } template void reset_except(View const& view) { - if (view.data() != internal_view.data()) reset(); + reset_except(execution_space(), view); + } + + template + void reset_except(const execution_space& exec_space, + View const& view) { + if (view.data() != internal_view.data()) reset(exec_space); } void resize(const size_t n0 = 0, const size_t n1 = 0, const size_t n2 = 0, @@ -928,10 +970,16 @@ class ScatterView ScatterView(View const& original_view) + : ScatterView(execution_space(), original_view) {} + + template + ScatterView(execution_space const& exec_space, + View const& original_view) : unique_token(), internal_view( view_alloc(WithoutInitializing, - std::string("duplicated_") + original_view.label()), + std::string("duplicated_") + original_view.label(), + exec_space), unique_token.size(), original_view.rank_dynamic > 0 ? original_view.extent(0) : KOKKOS_IMPL_CTOR_DEFAULT_ARG, @@ -949,14 +997,32 @@ class ScatterView ScatterView(std::string const& name, Dims... dims) - : internal_view(view_alloc(WithoutInitializing, name), + : ScatterView(view_alloc(execution_space(), name), dims...) {} + + // This overload allows specifying an execution space instance to be + // used by passing, e.g., Kokkos::view_alloc(exec_space, "label") as + // first argument. + template + ScatterView(::Kokkos::Impl::ViewCtorProp const& arg_prop, Dims... dims) + : internal_view(view_alloc(WithoutInitializing, + static_cast<::Kokkos::Impl::ViewCtorProp< + void, std::string> const&>(arg_prop) + .value), unique_token.size(), dims...) { - reset(); + using ::Kokkos::Impl::Experimental:: + check_scatter_view_allocation_properties_argument; + check_scatter_view_allocation_properties_argument(arg_prop); + + auto const exec_space = + static_cast<::Kokkos::Impl::ViewCtorProp const&>( + arg_prop) + .value; + reset(exec_space); } template @@ -984,37 +1050,51 @@ class ScatterView void contribute_into(View const& dest) const { + contribute_into(execution_space(), dest); + } + + template + void contribute_into(execution_space const& exec_space, + View const& dest) const { using dest_type = View; static_assert(std::is_same::value, "ScatterView deep_copy destination has different layout"); static_assert( - Kokkos::Impl::VerifyExecutionCanAccessMemorySpace< - memory_space, typename dest_type::memory_space>::value, + Kokkos::Impl::SpaceAccessibility< + execution_space, typename dest_type::memory_space>::accessible, "ScatterView deep_copy destination memory space not accessible"); bool is_equal = (dest.data() == internal_view.data()); size_t start = is_equal ? 1 : 0; Kokkos::Impl::Experimental::ReduceDuplicates( - internal_view.data(), dest.data(), internal_view.stride(0), start, - internal_view.extent(0), internal_view.label()); + exec_space, internal_view.data(), dest.data(), internal_view.stride(0), + start, internal_view.extent(0), internal_view.label()); } - void reset() { + void reset(execution_space const& exec_space = execution_space()) { Kokkos::Impl::Experimental::ResetDuplicates( - internal_view.data(), internal_view.size(), internal_view.label()); + exec_space, internal_view.data(), internal_view.size(), + internal_view.label()); } + template void reset_except(View const& view) { + reset_except(execution_space(), view); + } + + template + void reset_except(execution_space const& exec_space, + View const& view) { if (view.data() != internal_view.data()) { - reset(); + reset(exec_space); return; } Kokkos::Impl::Experimental::ResetDuplicates( - internal_view.data() + view.size(), internal_view.size() - view.size(), - internal_view.label()); + exec_space, internal_view.data() + view.size(), + internal_view.size() - view.size(), internal_view.label()); } void resize(const size_t n0 = 0, const size_t n1 = 0, const size_t n2 = 0, @@ -1075,7 +1155,13 @@ class ScatterView - ScatterView(View const& original_view) : unique_token() { + ScatterView(View const& original_view) + : ScatterView(execution_space(), original_view) {} + + template + ScatterView(execution_space const& exec_space, + View const& original_view) + : unique_token() { size_t arg_N[8] = {original_view.rank > 0 ? original_view.extent(0) : KOKKOS_IMPL_CTOR_DEFAULT_ARG, original_view.rank > 1 ? original_view.extent(1) @@ -1094,14 +1180,27 @@ class ScatterView - ScatterView(std::string const& name, Dims... dims) { + ScatterView(std::string const& name, Dims... dims) + : ScatterView(view_alloc(execution_space(), name), dims...) {} + + // This overload allows specifying an execution space instance to be + // used by passing, e.g., Kokkos::view_alloc(exec_space, "label") as + // first argument. + template + ScatterView(::Kokkos::Impl::ViewCtorProp const& arg_prop, + Dims... dims) { + using ::Kokkos::Impl::Experimental:: + check_scatter_view_allocation_properties_argument; + check_scatter_view_allocation_properties_argument(arg_prop); + original_view_type original_view; size_t arg_N[8] = {original_view.rank > 0 ? original_view.static_extent(0) : KOKKOS_IMPL_CTOR_DEFAULT_ARG, @@ -1120,10 +1219,20 @@ class ScatterView const&>( + arg_prop) + .value; internal_view = internal_view_type(view_alloc(WithoutInitializing, name), arg_N[0], arg_N[1], arg_N[2], arg_N[3], arg_N[4], arg_N[5], arg_N[6], arg_N[7]); - reset(); + + auto const exec_space = + static_cast<::Kokkos::Impl::ViewCtorProp const&>( + arg_prop) + .value; + reset(exec_space); } template @@ -1166,6 +1275,12 @@ class ScatterView void contribute_into(View const& dest) const { + contribute_into(execution_space(), dest); + } + + template + void contribute_into(execution_space const& exec_space, + View const& dest) const { using dest_type = View; static_assert( std::is_same::value, "ScatterView deep_copy destination has different layout"); static_assert( - Kokkos::Impl::VerifyExecutionCanAccessMemorySpace< - memory_space, typename dest_type::memory_space>::value, + Kokkos::Impl::SpaceAccessibility< + execution_space, typename dest_type::memory_space>::accessible, "ScatterView deep_copy destination memory space not accessible"); auto extent = internal_view.extent(internal_view_type::rank - 1); bool is_equal = (dest.data() == internal_view.data()); size_t start = is_equal ? 1 : 0; Kokkos::Impl::Experimental::ReduceDuplicates( - internal_view.data(), dest.data(), + exec_space, internal_view.data(), dest.data(), internal_view.stride(internal_view_type::rank - 1), start, extent, internal_view.label()); } - void reset() { + void reset(execution_space const& exec_space = execution_space()) { Kokkos::Impl::Experimental::ResetDuplicates( - internal_view.data(), internal_view.size(), internal_view.label()); + exec_space, internal_view.data(), internal_view.size(), + internal_view.label()); } + template void reset_except(View const& view) { + reset_except(execution_space(), view); + } + + template + void reset_except(execution_space const& exec_space, + View const& view) { if (view.data() != internal_view.data()) { - reset(); + reset(exec_space); return; } Kokkos::Impl::Experimental::ResetDuplicates( - internal_view.data() + view.size(), internal_view.size() - view.size(), - internal_view.label()); + exec_space, internal_view.data() + view.size(), + internal_view.size() - view.size(), internal_view.label()); } void resize(const size_t n0 = 0, const size_t n1 = 0, const size_t n2 = 0, @@ -1316,21 +1439,21 @@ template ::array_layout, typename ViewTraits::device_type, Op, - typename Kokkos::Impl::if_c< + std::conditional_t< std::is_same::value, typename Kokkos::Impl::Experimental::DefaultDuplication< typename ViewTraits::execution_space>::type, - Duplication>::type, - typename Kokkos::Impl::if_c< + Duplication>, + std::conditional_t< std::is_same::value, typename Kokkos::Impl::Experimental::DefaultContribution< typename ViewTraits::execution_space, - typename Kokkos::Impl::if_c< + typename std::conditional_t< std::is_same::value, typename Kokkos::Impl::Experimental::DefaultDuplication< typename ViewTraits::execution_space>::type, - Duplication>::type>::type, - Contribution>::type> + Duplication>>::type, + Contribution>> create_scatter_view(View const& original_view) { return original_view; // implicit ScatterView constructor call } @@ -1365,12 +1488,21 @@ create_scatter_view(Op, Duplication, Contribution, namespace Kokkos { namespace Experimental { +template +void contribute( + typename ES::execution_space const& exec_space, View& dest, + Kokkos::Experimental::ScatterView const& src) { + src.contribute_into(exec_space, dest); +} + template void contribute( View& dest, Kokkos::Experimental::ScatterView const& src) { - src.contribute_into(dest); + using execution_space = typename ES::execution_space; + contribute(execution_space{}, dest, src); } } // namespace Experimental diff --git a/lib/kokkos/containers/src/Kokkos_UnorderedMap.hpp b/lib/kokkos/containers/src/Kokkos_UnorderedMap.hpp index d2affda93a..edb0e7261d 100644 --- a/lib/kokkos/containers/src/Kokkos_UnorderedMap.hpp +++ b/lib/kokkos/containers/src/Kokkos_UnorderedMap.hpp @@ -264,26 +264,24 @@ class UnorderedMap { private: enum : size_type { invalid_index = ~static_cast(0) }; - using impl_value_type = - typename Impl::if_c::type; + using impl_value_type = std::conditional_t; - using key_type_view = typename Impl::if_c< + using key_type_view = std::conditional_t< is_insertable_map, View, - View > >::type; + View > >; - using value_type_view = - typename Impl::if_c, - View > >::type; + using value_type_view = std::conditional_t< + is_insertable_map || is_modifiable_map, + View, + View > >; - using size_type_view = typename Impl::if_c< + using size_type_view = std::conditional_t< is_insertable_map, View, - View > >::type; + View > >; using bitset_type = - typename Impl::if_c, - ConstBitset >::type; + std::conditional_t, + ConstBitset >; enum { modified_idx = 0, erasable_idx = 1, failed_insert_idx = 2 }; enum { num_scalars = 3 }; @@ -540,10 +538,7 @@ class UnorderedMap { // Previously claimed an unused entry that was not inserted. // Release this unused entry immediately. if (!m_available_indexes.reset(new_index)) { - // FIXME_SYCL SYCL doesn't allow printf in kernels -#ifndef KOKKOS_ENABLE_SYCL - printf("Unable to free existing\n"); -#endif + KOKKOS_IMPL_DO_NOT_USE_PRINTF("Unable to free existing\n"); } } @@ -659,8 +654,8 @@ class UnorderedMap { /// /// 'const value_type' via Cuda texture fetch must return by value. KOKKOS_FORCEINLINE_FUNCTION - typename Impl::if_c<(is_set || has_const_value), impl_value_type, - impl_value_type &>::type + std::conditional_t<(is_set || has_const_value), impl_value_type, + impl_value_type &> value_at(size_type i) const { return m_values[is_set ? 0 : (i < capacity() ? i : capacity())]; } diff --git a/lib/kokkos/containers/src/impl/Kokkos_Bitset_impl.hpp b/lib/kokkos/containers/src/impl/Kokkos_Bitset_impl.hpp index 6e450598d1..6047e60f3d 100644 --- a/lib/kokkos/containers/src/impl/Kokkos_Bitset_impl.hpp +++ b/lib/kokkos/containers/src/impl/Kokkos_Bitset_impl.hpp @@ -57,10 +57,22 @@ namespace Kokkos { namespace Impl { +KOKKOS_FORCEINLINE_FUNCTION +unsigned rotate_left(unsigned i, int r) { + constexpr int size = static_cast(sizeof(unsigned) * CHAR_BIT); + return r ? ((i << r) | (i >> (size - r))) : i; +} + KOKKOS_FORCEINLINE_FUNCTION unsigned rotate_right(unsigned i, int r) { - enum { size = static_cast(sizeof(unsigned) * CHAR_BIT) }; + constexpr int size = static_cast(sizeof(unsigned) * CHAR_BIT); + // FIXME_SYCL llvm.fshr.i32 missing + // (https://github.com/intel/llvm/issues/3308) +#ifdef __SYCL_DEVICE_ONLY__ + return rotate_left(i, size - r); +#else return r ? ((i >> r) | (i << (size - r))) : i; +#endif } template diff --git a/lib/kokkos/containers/src/impl/Kokkos_UnorderedMap_impl.hpp b/lib/kokkos/containers/src/impl/Kokkos_UnorderedMap_impl.hpp index b06ab0846c..d7c4a5d1ff 100644 --- a/lib/kokkos/containers/src/impl/Kokkos_UnorderedMap_impl.hpp +++ b/lib/kokkos/containers/src/impl/Kokkos_UnorderedMap_impl.hpp @@ -250,8 +250,8 @@ struct UnorderedMapPrint { uint32_t list = m_map.m_hash_lists(i); for (size_type curr = list, ii = 0; curr != invalid_index; curr = m_map.m_next_index[curr], ++ii) { - printf("%d[%d]: %d->%d\n", list, ii, m_map.key_at(curr), - m_map.value_at(curr)); + KOKKOS_IMPL_DO_NOT_USE_PRINTF("%d[%d]: %d->%d\n", list, ii, + m_map.key_at(curr), m_map.value_at(curr)); } } }; diff --git a/lib/kokkos/containers/unit_tests/CMakeLists.txt b/lib/kokkos/containers/unit_tests/CMakeLists.txt index c84c5f6d5e..947d222c27 100644 --- a/lib/kokkos/containers/unit_tests/CMakeLists.txt +++ b/lib/kokkos/containers/unit_tests/CMakeLists.txt @@ -2,6 +2,7 @@ KOKKOS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) KOKKOS_INCLUDE_DIRECTORIES(REQUIRED_DURING_INSTALLATION_TESTING ${CMAKE_CURRENT_SOURCE_DIR}) KOKKOS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../src ) +KOKKOS_INCLUDE_DIRECTORIES(${KOKKOS_SOURCE_DIR}/core/unit_test/category_files) foreach(Tag Threads;Serial;OpenMP;HPX;Cuda;HIP;SYCL) # Because there is always an exception to the rule @@ -41,11 +42,6 @@ foreach(Tag Threads;Serial;OpenMP;HPX;Cuda;HIP;SYCL) configure_file(${dir}/dummy.cpp ${file}) list(APPEND UnitTestSources ${file}) endforeach() - list(REMOVE_ITEM UnitTestSources - ${CMAKE_CURRENT_BINARY_DIR}/sycl/TestSYCL_Bitset.cpp - ${CMAKE_CURRENT_BINARY_DIR}/sycl/TestSYCL_ScatterView.cpp - ${CMAKE_CURRENT_BINARY_DIR}/sycl/TestSYCL_UnorderedMap.cpp - ) KOKKOS_ADD_EXECUTABLE_AND_TEST(UnitTest_${Tag} SOURCES ${UnitTestSources}) endif() endforeach() diff --git a/lib/kokkos/containers/unit_tests/Makefile b/lib/kokkos/containers/unit_tests/Makefile index f42b9b7519..82669fe1ab 100644 --- a/lib/kokkos/containers/unit_tests/Makefile +++ b/lib/kokkos/containers/unit_tests/Makefile @@ -26,7 +26,7 @@ override LDFLAGS += -lpthread include $(KOKKOS_PATH)/Makefile.kokkos -KOKKOS_CXXFLAGS += -I$(GTEST_PATH) -I${KOKKOS_PATH}/containers/unit_tests +KOKKOS_CXXFLAGS += -I$(GTEST_PATH) -I${KOKKOS_PATH}/containers/unit_tests -I${KOKKOS_PATH}/core/unit_test/category_files TEST_TARGETS = TARGETS = diff --git a/lib/kokkos/containers/unit_tests/TestCuda_Category.hpp b/lib/kokkos/containers/unit_tests/TestCuda_Category.hpp deleted file mode 100644 index 50935d7a34..0000000000 --- a/lib/kokkos/containers/unit_tests/TestCuda_Category.hpp +++ /dev/null @@ -1,51 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_TEST_CUDA_HPP -#define KOKKOS_TEST_CUDA_HPP - -#define TEST_CATEGORY cuda -#define TEST_EXECSPACE Kokkos::Cuda - -#endif diff --git a/lib/kokkos/containers/unit_tests/TestDualView.hpp b/lib/kokkos/containers/unit_tests/TestDualView.hpp index 531caf0f85..3eee85ed10 100644 --- a/lib/kokkos/containers/unit_tests/TestDualView.hpp +++ b/lib/kokkos/containers/unit_tests/TestDualView.hpp @@ -114,6 +114,8 @@ struct test_dualview_combinations { a.template modify(); a.template sync(); + a.template sync( + Kokkos::DefaultExecutionSpace{}); a.h_view(5, 1) = 3; a.h_view(6, 1) = 4; @@ -122,11 +124,15 @@ struct test_dualview_combinations { ViewType b = Kokkos::subview(a, std::pair(6, 9), std::pair(0, 1)); a.template sync(); + a.template sync( + Kokkos::DefaultExecutionSpace{}); b.template modify(); Kokkos::deep_copy(b.d_view, 2); a.template sync(); + a.template sync( + Kokkos::DefaultExecutionSpace{}); Scalar count = 0; for (unsigned int i = 0; i < a.d_view.extent(0); i++) for (unsigned int j = 0; j < a.d_view.extent(1); j++) @@ -180,6 +186,7 @@ struct test_dual_view_deep_copy { } else { a.modify_device(); a.sync_host(); + a.sync_host(Kokkos::DefaultExecutionSpace{}); } // Check device view is initialized as expected @@ -208,6 +215,7 @@ struct test_dual_view_deep_copy { b.template sync(); } else { b.sync_host(); + b.sync_host(Kokkos::DefaultExecutionSpace{}); } // Perform same checks on b as done on a @@ -302,6 +310,7 @@ struct test_dualview_resize { ASSERT_EQ(a.extent(1), m / factor); a.sync_device(); + a.sync_device(Kokkos::DefaultExecutionSpace{}); // Check device view is initialized as expected a_d_sum = 0; @@ -404,19 +413,14 @@ void test_dualview_resize() { Impl::test_dualview_resize(); } -// FIXME_SYCL requires MDRange policy -#ifndef KOKKOS_ENABLE_SYCL TEST(TEST_CATEGORY, dualview_combination) { test_dualview_combinations(10, true); } -#endif TEST(TEST_CATEGORY, dualview_alloc) { test_dualview_alloc(10); } -// FIXME_SYCL requires MDRange policy -#ifndef KOKKOS_ENABLE_SYCL TEST(TEST_CATEGORY, dualview_combinations_without_init) { test_dualview_combinations(10, false); } @@ -433,8 +437,133 @@ TEST(TEST_CATEGORY, dualview_realloc) { TEST(TEST_CATEGORY, dualview_resize) { test_dualview_resize(); } + +namespace { +/** + * + * The following tests are a response to + * https://github.com/kokkos/kokkos/issues/3850 + * and + * https://github.com/kokkos/kokkos/pull/3857 + * + * DualViews were returning incorrect view types and taking + * inappropriate actions based on the templated view methods. + * + * Specifically, template view methods were always returning + * a device view if the memory space was UVM and a Kokkos::Device was passed. + * Sync/modify methods completely broke down So these tests exist to make sure + * that we keep the semantics of UVM DualViews intact. + */ +// modify if we have other UVM enabled backends +#ifdef KOKKOS_ENABLE_CUDA // OR other UVM builds +#define UVM_ENABLED_BUILD #endif +#ifdef UVM_ENABLED_BUILD +template +struct UVMSpaceFor; +#endif + +#ifdef KOKKOS_ENABLE_CUDA // specific to CUDA +template <> +struct UVMSpaceFor { + using type = Kokkos::CudaUVMSpace; +}; +#endif + +#ifdef UVM_ENABLED_BUILD +template <> +struct UVMSpaceFor { + using type = typename UVMSpaceFor::type; +}; +#else +template +struct UVMSpaceFor { + using type = typename ExecSpace::memory_space; +}; +#endif + +using ExecSpace = Kokkos::DefaultExecutionSpace; +using MemSpace = typename UVMSpaceFor::type; +using DeviceType = Kokkos::Device; + +using DualViewType = Kokkos::DualView; +using d_device = DeviceType; +using h_device = Kokkos::Device< + Kokkos::DefaultHostExecutionSpace, + typename UVMSpaceFor::type>; + +TEST(TEST_CATEGORY, dualview_device_correct_kokkos_device) { + DualViewType dv("myView", 100); + dv.clear_sync_state(); + auto v_d = dv.template view(); + using vdt = decltype(v_d); + using vdt_d = vdt::device_type; + using vdt_d_e = vdt_d::execution_space; + ASSERT_STREQ(vdt_d_e::name(), Kokkos::DefaultExecutionSpace::name()); +} +TEST(TEST_CATEGORY, dualview_host_correct_kokkos_device) { + DualViewType dv("myView", 100); + dv.clear_sync_state(); + auto v_h = dv.template view(); + using vht = decltype(v_h); + using vht_d = vht::device_type; + using vht_d_e = vht_d::execution_space; + ASSERT_STREQ(vht_d_e::name(), Kokkos::DefaultHostExecutionSpace::name()); +} + +TEST(TEST_CATEGORY, dualview_host_modify_template_device_sync) { + DualViewType dv("myView", 100); + dv.clear_sync_state(); + dv.modify_host(); + dv.template sync(); + EXPECT_TRUE(!dv.need_sync_device()); + EXPECT_TRUE(!dv.need_sync_host()); + dv.clear_sync_state(); +} + +TEST(TEST_CATEGORY, dualview_host_modify_template_device_execspace_sync) { + DualViewType dv("myView", 100); + dv.clear_sync_state(); + dv.modify_host(); + dv.template sync(); + EXPECT_TRUE(!dv.need_sync_device()); + EXPECT_TRUE(!dv.need_sync_host()); + dv.clear_sync_state(); +} + +TEST(TEST_CATEGORY, dualview_device_modify_template_host_sync) { + DualViewType dv("myView", 100); + dv.clear_sync_state(); + dv.modify_device(); + dv.template sync(); + EXPECT_TRUE(!dv.need_sync_device()); + EXPECT_TRUE(!dv.need_sync_host()); + dv.clear_sync_state(); +} +TEST(TEST_CATEGORY, dualview_device_modify_template_host_execspace_sync) { + DualViewType dv("myView", 100); + dv.clear_sync_state(); + dv.modify_device(); + dv.template sync(); + EXPECT_TRUE(!dv.need_sync_device()); + EXPECT_TRUE(!dv.need_sync_host()); + dv.clear_sync_state(); +} + +TEST(TEST_CATEGORY, + dualview_template_views_return_correct_executionspace_views) { + DualViewType dv("myView", 100); + dv.clear_sync_state(); + using hvt = decltype(dv.view()); + using dvt = decltype(dv.view()); + ASSERT_STREQ(Kokkos::DefaultExecutionSpace::name(), + dvt::device_type::execution_space::name()); + ASSERT_STREQ(Kokkos::DefaultHostExecutionSpace::name(), + hvt::device_type::execution_space::name()); +} + +} // anonymous namespace } // namespace Test #endif // KOKKOS_TEST_DUALVIEW_HPP diff --git a/lib/kokkos/containers/unit_tests/TestDynamicView.hpp b/lib/kokkos/containers/unit_tests/TestDynamicView.hpp index 4b9f994417..f018793dd6 100644 --- a/lib/kokkos/containers/unit_tests/TestDynamicView.hpp +++ b/lib/kokkos/containers/unit_tests/TestDynamicView.hpp @@ -243,8 +243,6 @@ struct TestDynamicView { } }; -// FIXME_SYCL needs resize_serial -#ifndef KOKKOS_ENABLE_SYCL TEST(TEST_CATEGORY, dynamic_view) { using TestDynView = TestDynamicView; @@ -252,7 +250,6 @@ TEST(TEST_CATEGORY, dynamic_view) { TestDynView::run(100000 + 100 * i); } } -#endif } // namespace Test diff --git a/lib/kokkos/containers/unit_tests/TestHPX_Category.hpp b/lib/kokkos/containers/unit_tests/TestHPX_Category.hpp deleted file mode 100644 index 64fc7c0757..0000000000 --- a/lib/kokkos/containers/unit_tests/TestHPX_Category.hpp +++ /dev/null @@ -1,51 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_TEST_HPX_HPP -#define KOKKOS_TEST_HPX_HPP - -#define TEST_CATEGORY hpx -#define TEST_EXECSPACE Kokkos::Experimental::HPX - -#endif diff --git a/lib/kokkos/containers/unit_tests/TestOffsetView.hpp b/lib/kokkos/containers/unit_tests/TestOffsetView.hpp index 802813b13b..9ddc226e29 100644 --- a/lib/kokkos/containers/unit_tests/TestOffsetView.hpp +++ b/lib/kokkos/containers/unit_tests/TestOffsetView.hpp @@ -130,8 +130,6 @@ void test_offsetview_construction() { } } - // FIXME_SYCL requires MDRange policy -#ifndef KOKKOS_ENABLE_SYCL const int ovmin0 = ov.begin(0); const int ovend0 = ov.end(0); const int ovmin1 = ov.begin(1); @@ -178,7 +176,6 @@ void test_offsetview_construction() { } ASSERT_EQ(OVResult, answer) << "Bad data found in OffsetView"; -#endif #endif { @@ -215,8 +212,6 @@ void test_offsetview_construction() { point3_type{{extent0, extent1, extent2}}); #if defined(KOKKOS_ENABLE_CUDA_LAMBDA) || !defined(KOKKOS_ENABLE_CUDA) - // FIXME_SYCL requires MDRange policy -#ifdef KOKKOS_ENABLE_SYCL int view3DSum = 0; Kokkos::parallel_reduce( rangePolicy3DZero, @@ -239,7 +234,6 @@ void test_offsetview_construction() { ASSERT_EQ(view3DSum, offsetView3DSum) << "construction of OffsetView from View and begins array broken."; -#endif #endif } view_type viewFromOV = ov.view(); @@ -266,8 +260,6 @@ void test_offsetview_construction() { Kokkos::deep_copy(aView, ov); #if defined(KOKKOS_ENABLE_CUDA_LAMBDA) || !defined(KOKKOS_ENABLE_CUDA) - // FIXME_SYCL requires MDRange policy -#ifndef KOKKOS_ENABLE_SYCL int sum = 0; Kokkos::parallel_reduce( rangePolicy2D, @@ -277,7 +269,6 @@ void test_offsetview_construction() { sum); ASSERT_EQ(sum, 0) << "deep_copy(view, offsetView) broken."; -#endif #endif } @@ -288,8 +279,6 @@ void test_offsetview_construction() { Kokkos::deep_copy(ov, aView); #if defined(KOKKOS_ENABLE_CUDA_LAMBDA) || !defined(KOKKOS_ENABLE_CUDA) - // FIXME_SYCL requires MDRange policy -#ifndef KOKKOS_ENABLE_SYCL int sum = 0; Kokkos::parallel_reduce( rangePolicy2D, @@ -299,7 +288,6 @@ void test_offsetview_construction() { sum); ASSERT_EQ(sum, 0) << "deep_copy(offsetView, view) broken."; -#endif #endif } } @@ -471,8 +459,6 @@ void test_offsetview_subview() { ASSERT_EQ(offsetSubview.end(1), 9); #if defined(KOKKOS_ENABLE_CUDA_LAMBDA) || !defined(KOKKOS_ENABLE_CUDA) - // FIXME_SYCL requires MDRange policy -#ifndef KOKKOS_ENABLE_SYCL using range_type = Kokkos::MDRangePolicy, Kokkos::IndexType >; using point_type = typename range_type::point_type; @@ -498,7 +484,6 @@ void test_offsetview_subview() { sum); ASSERT_EQ(sum, 6 * (e0 - b0) * (e1 - b1)); -#endif #endif } @@ -701,12 +686,9 @@ void test_offsetview_offsets_rank3() { } #endif -// FIXME_SYCL needs MDRangePolicy -#ifndef KOKKOS_ENABLE_SYCL TEST(TEST_CATEGORY, offsetview_construction) { test_offsetview_construction(); } -#endif TEST(TEST_CATEGORY, offsetview_unmanaged_construction) { test_offsetview_unmanaged_construction(); diff --git a/lib/kokkos/containers/unit_tests/TestOpenMP_Category.hpp b/lib/kokkos/containers/unit_tests/TestOpenMP_Category.hpp deleted file mode 100644 index a0169d1702..0000000000 --- a/lib/kokkos/containers/unit_tests/TestOpenMP_Category.hpp +++ /dev/null @@ -1,51 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_TEST_OPENMP_HPP -#define KOKKOS_TEST_OPENMP_HPP - -#define TEST_CATEGORY openmp -#define TEST_EXECSPACE Kokkos::OpenMP - -#endif diff --git a/lib/kokkos/containers/unit_tests/TestSYCL_Category.hpp b/lib/kokkos/containers/unit_tests/TestSYCL_Category.hpp deleted file mode 100644 index 51fd3fc911..0000000000 --- a/lib/kokkos/containers/unit_tests/TestSYCL_Category.hpp +++ /dev/null @@ -1,51 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_TEST_SYCL_HPP -#define KOKKOS_TEST_SYCL_HPP - -#define TEST_CATEGORY sycl -#define TEST_EXECSPACE Kokkos::Experimental::SYCL - -#endif diff --git a/lib/kokkos/containers/unit_tests/TestScatterView.hpp b/lib/kokkos/containers/unit_tests/TestScatterView.hpp index 3a3cb607a6..fdbce2d492 100644 --- a/lib/kokkos/containers/unit_tests/TestScatterView.hpp +++ b/lib/kokkos/containers/unit_tests/TestScatterView.hpp @@ -437,6 +437,10 @@ struct test_scatter_view_config { Contribution, Op, NumberType>::orig_view_type; + void compile_constructor() { + auto sv = scatter_view_def(Kokkos::view_alloc(DeviceType{}, "label"), 10); + } + void run_test(int n) { // test allocation { diff --git a/lib/kokkos/containers/unit_tests/TestSerial_Category.hpp b/lib/kokkos/containers/unit_tests/TestSerial_Category.hpp deleted file mode 100644 index 2aa09a315a..0000000000 --- a/lib/kokkos/containers/unit_tests/TestSerial_Category.hpp +++ /dev/null @@ -1,51 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_TEST_SERIAL_HPP -#define KOKKOS_TEST_SERIAL_HPP - -#define TEST_CATEGORY serial -#define TEST_EXECSPACE Kokkos::Serial - -#endif diff --git a/lib/kokkos/containers/unit_tests/TestStaticCrsGraph.hpp b/lib/kokkos/containers/unit_tests/TestStaticCrsGraph.hpp index 8bb267ce5d..a9a178f95e 100644 --- a/lib/kokkos/containers/unit_tests/TestStaticCrsGraph.hpp +++ b/lib/kokkos/containers/unit_tests/TestStaticCrsGraph.hpp @@ -285,10 +285,7 @@ void run_test_graph4() { TEST(TEST_CATEGORY, staticcrsgraph) { TestStaticCrsGraph::run_test_graph(); - // FIXME_SYCL requires MDRangePolicy -#ifndef KOKKOS_ENABLE_SYCL TestStaticCrsGraph::run_test_graph2(); -#endif TestStaticCrsGraph::run_test_graph3(1, 0); TestStaticCrsGraph::run_test_graph3(1, 1000); TestStaticCrsGraph::run_test_graph3(1, 10000); diff --git a/lib/kokkos/containers/unit_tests/TestThreads_Category.hpp b/lib/kokkos/containers/unit_tests/TestThreads_Category.hpp deleted file mode 100644 index 74a2b0da36..0000000000 --- a/lib/kokkos/containers/unit_tests/TestThreads_Category.hpp +++ /dev/null @@ -1,51 +0,0 @@ -/* -//@HEADER -// ************************************************************************ -// -// Kokkos v. 3.0 -// Copyright (2020) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact Christian R. Trott (crtrott@sandia.gov) -// -// ************************************************************************ -//@HEADER -*/ - -#ifndef KOKKOS_TEST_THREADS_HPP -#define KOKKOS_TEST_THREADS_HPP - -#define TEST_CATEGORY threads -#define TEST_EXECSPACE Kokkos::Threads - -#endif diff --git a/lib/kokkos/containers/unit_tests/TestUnorderedMap.hpp b/lib/kokkos/containers/unit_tests/TestUnorderedMap.hpp index d39e0061c7..4413cfbc80 100644 --- a/lib/kokkos/containers/unit_tests/TestUnorderedMap.hpp +++ b/lib/kokkos/containers/unit_tests/TestUnorderedMap.hpp @@ -163,7 +163,8 @@ struct TestFind { KOKKOS_INLINE_FUNCTION void operator()(typename execution_space::size_type i, value_type &errors) const { - const bool expect_to_find_i = (i < m_max_key); + const bool expect_to_find_i = + (i < typename execution_space::size_type(m_max_key)); const bool exists = m_map.exists(i); @@ -293,10 +294,11 @@ void test_deep_copy(uint32_t num_nodes) { } } -// FIXME_HIP wrong result in CI but works locally -#ifndef KOKKOS_ENABLE_HIP +// FIXME_SYCL wrong results on Nvidia GPUs but correct on Host and Intel GPUs +// FIXME_HIP // WORKAROUND MSVC -#ifndef _WIN32 +#if !(defined(KOKKOS_ENABLE_HIP) && (HIP_VERSION < 401)) && \ + !defined(_WIN32) && !defined(KOKKOS_ENABLE_SYCL) TEST(TEST_CATEGORY, UnorderedMap_insert) { for (int i = 0; i < 500; ++i) { test_insert(100000, 90000, 100, true); @@ -304,7 +306,6 @@ TEST(TEST_CATEGORY, UnorderedMap_insert) { } } #endif -#endif TEST(TEST_CATEGORY, UnorderedMap_failed_insert) { for (int i = 0; i < 1000; ++i) test_failed_insert(10000); diff --git a/lib/kokkos/core/perf_test/CMakeLists.txt b/lib/kokkos/core/perf_test/CMakeLists.txt index b7b817c910..9ff4b6006d 100644 --- a/lib/kokkos/core/perf_test/CMakeLists.txt +++ b/lib/kokkos/core/perf_test/CMakeLists.txt @@ -9,6 +9,14 @@ # that in TriBITS KokkosAlgorithms can be disabled... #INCLUDE_DIRECTORIES("${CMAKE_CURRENT_SOURCE_DIR}/../../algorithms/src") +# FIXME_OPENMPTARGET - the NVIDIA HPC compiler nvc++ in the OpenMPTarget backend does not pass the perf_tests. +IF (KOKKOS_ENABLE_OPENMPTARGET + AND (KOKKOS_CXX_COMPILER_ID STREQUAL PGI + OR KOKKOS_CXX_COMPILER_ID STREQUAL NVHPC)) + RETURN() +ENDIF() + + SET(SOURCES PerfTestMain.cpp PerfTestGramSchmidt.cpp @@ -68,8 +76,7 @@ KOKKOS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) KOKKOS_INCLUDE_DIRECTORIES(REQUIRED_DURING_INSTALLATION_TESTING ${CMAKE_CURRENT_SOURCE_DIR}) # This test currently times out for MSVC -# FIXME_SYCL these tests don't compile yet (require parallel_for). -IF(NOT KOKKOS_CXX_COMPILER_ID STREQUAL "MSVC" AND NOT Kokkos_ENABLE_SYCL) +IF(NOT KOKKOS_CXX_COMPILER_ID STREQUAL "MSVC") KOKKOS_ADD_EXECUTABLE_AND_TEST( PerfTestExec SOURCES ${SOURCES} @@ -77,13 +84,11 @@ IF(NOT KOKKOS_CXX_COMPILER_ID STREQUAL "MSVC" AND NOT Kokkos_ENABLE_SYCL) ) ENDIF() -# FIXME_SYCL -IF(NOT Kokkos_ENABLE_SYCL) - KOKKOS_ADD_EXECUTABLE_AND_TEST( - PerformanceTest_Atomic - SOURCES test_atomic.cpp - CATEGORIES PERFORMANCE - ) +KOKKOS_ADD_EXECUTABLE_AND_TEST( + PerformanceTest_Atomic + SOURCES test_atomic.cpp + CATEGORIES PERFORMANCE +) IF(NOT KOKKOS_ENABLE_CUDA OR KOKKOS_ENABLE_CUDA_LAMBDA) KOKKOS_ADD_EXECUTABLE_AND_TEST( @@ -98,7 +103,6 @@ KOKKOS_ADD_EXECUTABLE_AND_TEST( SOURCES test_mempool.cpp CATEGORIES PERFORMANCE ) -ENDIF() IF(NOT Kokkos_ENABLE_OPENMPTARGET) # FIXME OPENMPTARGET needs tasking diff --git a/lib/kokkos/core/perf_test/PerfTestGramSchmidt.cpp b/lib/kokkos/core/perf_test/PerfTestGramSchmidt.cpp index 70186283c1..dee21fd7a5 100644 --- a/lib/kokkos/core/perf_test/PerfTestGramSchmidt.cpp +++ b/lib/kokkos/core/perf_test/PerfTestGramSchmidt.cpp @@ -69,7 +69,7 @@ struct InvNorm2 : public Kokkos::DotSingle { KOKKOS_INLINE_FUNCTION void final(value_type& result) const { - result = std::sqrt(result); + result = Kokkos::Experimental::sqrt(result); Rjj() = result; inv() = (0 < result) ? 1.0 / result : 0; } @@ -145,7 +145,7 @@ struct ModifiedGramSchmidt { // Q(:,j) *= ( 1 / R(j,j) ); => Q(:,j) *= tmp ; Kokkos::scale(tmp, Qj); - for (size_t k = j + 1; k < count; ++k) { + for (size_type k = j + 1; k < count; ++k) { const vector_type Qk = Kokkos::subview(Q_, Kokkos::ALL(), k); const value_view Rjk = Kokkos::subview(R_, j, k); @@ -165,7 +165,7 @@ struct ModifiedGramSchmidt { //-------------------------------------------------------------------------- - static double test(const size_t length, const size_t count, + static double test(const size_type length, const size_type count, const size_t iter = 1) { multivector_type Q_("Q", length, count); multivector_type R_("R", count, count); diff --git a/lib/kokkos/core/src/CMakeLists.txt b/lib/kokkos/core/src/CMakeLists.txt index e0590a78a4..2ab0989805 100644 --- a/lib/kokkos/core/src/CMakeLists.txt +++ b/lib/kokkos/core/src/CMakeLists.txt @@ -72,8 +72,6 @@ KOKKOS_ADD_LIBRARY( ADD_BUILD_OPTIONS # core should be given all the necessary compiler/linker flags ) -SET_TARGET_PROPERTIES(kokkoscore PROPERTIES VERSION ${Kokkos_VERSION}) - KOKKOS_LIB_INCLUDE_DIRECTORIES(kokkoscore ${KOKKOS_TOP_BUILD_DIR} ${CMAKE_CURRENT_BINARY_DIR} @@ -87,3 +85,4 @@ KOKKOS_LINK_TPL(kokkoscore PUBLIC HPX) KOKKOS_LINK_TPL(kokkoscore PUBLIC LIBDL) KOKKOS_LINK_TPL(kokkoscore PUBLIC LIBRT) KOKKOS_LINK_TPL(kokkoscore PUBLIC PTHREAD) +KOKKOS_LINK_TPL(kokkoscore PUBLIC ROCM) diff --git a/lib/kokkos/core/src/Cuda/Kokkos_CudaSpace.cpp b/lib/kokkos/core/src/Cuda/Kokkos_CudaSpace.cpp index 4a30c914f0..916f109758 100644 --- a/lib/kokkos/core/src/Cuda/Kokkos_CudaSpace.cpp +++ b/lib/kokkos/core/src/Cuda/Kokkos_CudaSpace.cpp @@ -45,6 +45,10 @@ #include #ifdef KOKKOS_ENABLE_CUDA +#include +#include +#include + #include #include #include @@ -52,10 +56,6 @@ #include #include -#include -#include -#include - //#include #include #include @@ -65,6 +65,22 @@ /*--------------------------------------------------------------------------*/ /*--------------------------------------------------------------------------*/ +cudaStream_t Kokkos::Impl::cuda_get_deep_copy_stream() { + static cudaStream_t s = nullptr; + if (s == nullptr) { + cudaStreamCreate(&s); + } + return s; +} + +const std::unique_ptr &Kokkos::Impl::cuda_get_deep_copy_space( + bool initialize) { + static std::unique_ptr space = nullptr; + if (!space && initialize) + space = std::make_unique(Kokkos::Impl::cuda_get_deep_copy_stream()); + return space; +} + namespace Kokkos { namespace Impl { @@ -72,13 +88,6 @@ namespace { static std::atomic num_uvm_allocations(0); -cudaStream_t get_deep_copy_stream() { - static cudaStream_t s = nullptr; - if (s == nullptr) { - cudaStreamCreate(&s); - } - return s; -} } // namespace DeepCopy::DeepCopy(void *dst, const void *src, @@ -115,7 +124,7 @@ DeepCopy::DeepCopy(const Cuda &instance, void *dst, } void DeepCopyAsyncCuda(void *dst, const void *src, size_t n) { - cudaStream_t s = get_deep_copy_stream(); + cudaStream_t s = cuda_get_deep_copy_stream(); CUDA_SAFE_CALL(cudaMemcpyAsync(dst, src, n, cudaMemcpyDefault, s)); cudaStreamSynchronize(s); } @@ -128,14 +137,14 @@ void DeepCopyAsyncCuda(void *dst, const void *src, size_t n) { namespace Kokkos { -void CudaSpace::access_error() { +KOKKOS_DEPRECATED void CudaSpace::access_error() { const std::string msg( "Kokkos::CudaSpace::access_error attempt to execute Cuda function from " "non-Cuda space"); Kokkos::Impl::throw_runtime_exception(msg); } -void CudaSpace::access_error(const void *const) { +KOKKOS_DEPRECATED void CudaSpace::access_error(const void *const) { const std::string msg( "Kokkos::CudaSpace::access_error attempt to execute Cuda function from " "non-Cuda space"); @@ -459,79 +468,6 @@ SharedAllocationRecord::attach_texture_object( return tex_obj; } -//============================================================================== -// {{{1 - -std::string SharedAllocationRecord::get_label() const { - SharedAllocationHeader header; - - Kokkos::Impl::DeepCopy( - &header, RecordBase::head(), sizeof(SharedAllocationHeader)); - - return std::string(header.m_label); -} - -std::string SharedAllocationRecord::get_label() - const { - return std::string(RecordBase::head()->m_label); -} - -std::string -SharedAllocationRecord::get_label() const { - return std::string(RecordBase::head()->m_label); -} - -// end SharedAllocationRecord::get_label() }}}1 -//============================================================================== - -//============================================================================== -// {{{1 - -SharedAllocationRecord - *SharedAllocationRecord::allocate( - const Kokkos::CudaSpace &arg_space, const std::string &arg_label, - const size_t arg_alloc_size) { - return new SharedAllocationRecord(arg_space, arg_label, arg_alloc_size); -} - -SharedAllocationRecord - *SharedAllocationRecord::allocate( - const Kokkos::CudaUVMSpace &arg_space, const std::string &arg_label, - const size_t arg_alloc_size) { - return new SharedAllocationRecord(arg_space, arg_label, arg_alloc_size); -} - -SharedAllocationRecord - *SharedAllocationRecord::allocate( - const Kokkos::CudaHostPinnedSpace &arg_space, - const std::string &arg_label, const size_t arg_alloc_size) { - return new SharedAllocationRecord(arg_space, arg_label, arg_alloc_size); -} - -// end SharedAllocationRecord allocate() }}}1 -//============================================================================== - -//============================================================================== -// {{{1 - -void SharedAllocationRecord::deallocate( - SharedAllocationRecord *arg_rec) { - delete static_cast(arg_rec); -} - -void SharedAllocationRecord::deallocate( - SharedAllocationRecord *arg_rec) { - delete static_cast(arg_rec); -} - -void SharedAllocationRecord::deallocate( - SharedAllocationRecord *arg_rec) { - delete static_cast(arg_rec); -} - -// end SharedAllocationRecord deallocate }}}1 -//============================================================================== - //============================================================================== // {{{1 @@ -580,7 +516,7 @@ SharedAllocationRecord::SharedAllocationRecord( const SharedAllocationRecord::function_type arg_dealloc) // Pass through allocated [ SharedAllocationHeader , user_memory ] // Pass through deallocation function - : SharedAllocationRecord( + : base_t( #ifdef KOKKOS_ENABLE_DEBUG &SharedAllocationRecord::s_root_record, #endif @@ -592,13 +528,7 @@ SharedAllocationRecord::SharedAllocationRecord( SharedAllocationHeader header; - // Fill in the Header information - header.m_record = static_cast *>(this); - - strncpy(header.m_label, arg_label.c_str(), - SharedAllocationHeader::maximum_label_length); - // Set last element zero, in case c_str is too long - header.m_label[SharedAllocationHeader::maximum_label_length - 1] = (char)0; + this->base_t::_fill_host_accessible_header_info(header, arg_label); // Copy to device memory Kokkos::Impl::DeepCopy(RecordBase::m_alloc_ptr, &header, @@ -611,7 +541,7 @@ SharedAllocationRecord::SharedAllocationRecord( const SharedAllocationRecord::function_type arg_dealloc) // Pass through allocated [ SharedAllocationHeader , user_memory ] // Pass through deallocation function - : SharedAllocationRecord( + : base_t( #ifdef KOKKOS_ENABLE_DEBUG &SharedAllocationRecord::s_root_record, #endif @@ -620,16 +550,8 @@ SharedAllocationRecord::SharedAllocationRecord( sizeof(SharedAllocationHeader) + arg_alloc_size, arg_dealloc), m_tex_obj(0), m_space(arg_space) { - // Fill in the Header information, directly accessible via UVM - - RecordBase::m_alloc_ptr->m_record = this; - - strncpy(RecordBase::m_alloc_ptr->m_label, arg_label.c_str(), - SharedAllocationHeader::maximum_label_length); - - // Set last element zero, in case c_str is too long - RecordBase::m_alloc_ptr - ->m_label[SharedAllocationHeader::maximum_label_length - 1] = (char)0; + this->base_t::_fill_host_accessible_header_info(*base_t::m_alloc_ptr, + arg_label); } SharedAllocationRecord:: @@ -639,7 +561,7 @@ SharedAllocationRecord:: const SharedAllocationRecord::function_type arg_dealloc) // Pass through allocated [ SharedAllocationHeader , user_memory ] // Pass through deallocation function - : SharedAllocationRecord( + : base_t( #ifdef KOKKOS_ENABLE_DEBUG &SharedAllocationRecord::s_root_record, @@ -648,319 +570,13 @@ SharedAllocationRecord:: arg_alloc_size), sizeof(SharedAllocationHeader) + arg_alloc_size, arg_dealloc), m_space(arg_space) { - // Fill in the Header information, directly accessible on the host - - RecordBase::m_alloc_ptr->m_record = this; - - strncpy(RecordBase::m_alloc_ptr->m_label, arg_label.c_str(), - SharedAllocationHeader::maximum_label_length); - // Set last element zero, in case c_str is too long - RecordBase::m_alloc_ptr - ->m_label[SharedAllocationHeader::maximum_label_length - 1] = (char)0; + this->base_t::_fill_host_accessible_header_info(*base_t::m_alloc_ptr, + arg_label); } // end SharedAllocationRecord constructors }}}1 //============================================================================== -//============================================================================== -// {{{1 - -void *SharedAllocationRecord::allocate_tracked( - const Kokkos::CudaSpace &arg_space, const std::string &arg_alloc_label, - const size_t arg_alloc_size) { - if (!arg_alloc_size) return nullptr; - - SharedAllocationRecord *const r = - allocate(arg_space, arg_alloc_label, arg_alloc_size); - - RecordBase::increment(r); - - return r->data(); -} - -void SharedAllocationRecord::deallocate_tracked( - void *const arg_alloc_ptr) { - if (arg_alloc_ptr != nullptr) { - SharedAllocationRecord *const r = get_record(arg_alloc_ptr); - - RecordBase::decrement(r); - } -} - -void *SharedAllocationRecord::reallocate_tracked( - void *const arg_alloc_ptr, const size_t arg_alloc_size) { - SharedAllocationRecord *const r_old = get_record(arg_alloc_ptr); - SharedAllocationRecord *const r_new = - allocate(r_old->m_space, r_old->get_label(), arg_alloc_size); - - Kokkos::Impl::DeepCopy( - r_new->data(), r_old->data(), std::min(r_old->size(), r_new->size())); - - RecordBase::increment(r_new); - RecordBase::decrement(r_old); - - return r_new->data(); -} - -void *SharedAllocationRecord::allocate_tracked( - const Kokkos::CudaUVMSpace &arg_space, const std::string &arg_alloc_label, - const size_t arg_alloc_size) { - if (!arg_alloc_size) return nullptr; - - SharedAllocationRecord *const r = - allocate(arg_space, arg_alloc_label, arg_alloc_size); - - RecordBase::increment(r); - - return r->data(); -} - -void SharedAllocationRecord::deallocate_tracked( - void *const arg_alloc_ptr) { - if (arg_alloc_ptr != nullptr) { - SharedAllocationRecord *const r = get_record(arg_alloc_ptr); - - RecordBase::decrement(r); - } -} - -void *SharedAllocationRecord::reallocate_tracked( - void *const arg_alloc_ptr, const size_t arg_alloc_size) { - SharedAllocationRecord *const r_old = get_record(arg_alloc_ptr); - SharedAllocationRecord *const r_new = - allocate(r_old->m_space, r_old->get_label(), arg_alloc_size); - - Kokkos::Impl::DeepCopy( - r_new->data(), r_old->data(), std::min(r_old->size(), r_new->size())); - - RecordBase::increment(r_new); - RecordBase::decrement(r_old); - - return r_new->data(); -} - -void * -SharedAllocationRecord::allocate_tracked( - const Kokkos::CudaHostPinnedSpace &arg_space, - const std::string &arg_alloc_label, const size_t arg_alloc_size) { - if (!arg_alloc_size) return nullptr; - - SharedAllocationRecord *const r = - allocate(arg_space, arg_alloc_label, arg_alloc_size); - - RecordBase::increment(r); - - return r->data(); -} - -void SharedAllocationRecord::deallocate_tracked(void *const - arg_alloc_ptr) { - if (arg_alloc_ptr != nullptr) { - SharedAllocationRecord *const r = get_record(arg_alloc_ptr); - - RecordBase::decrement(r); - } -} - -void * -SharedAllocationRecord::reallocate_tracked( - void *const arg_alloc_ptr, const size_t arg_alloc_size) { - SharedAllocationRecord *const r_old = get_record(arg_alloc_ptr); - SharedAllocationRecord *const r_new = - allocate(r_old->m_space, r_old->get_label(), arg_alloc_size); - - Kokkos::Impl::DeepCopy( - r_new->data(), r_old->data(), std::min(r_old->size(), r_new->size())); - - RecordBase::increment(r_new); - RecordBase::decrement(r_old); - - return r_new->data(); -} - -// end SharedAllocationRecored::(re|de|)allocate_tracked }}}1 -//============================================================================== - -//============================================================================== -// {{{1 - -SharedAllocationRecord * -SharedAllocationRecord::get_record(void *alloc_ptr) { - using RecordCuda = SharedAllocationRecord; - - using Header = SharedAllocationHeader; - - // Copy the header from the allocation - Header head; - - Header const *const head_cuda = - alloc_ptr ? Header::get_header(alloc_ptr) : nullptr; - - if (alloc_ptr) { - Kokkos::Impl::DeepCopy( - &head, head_cuda, sizeof(SharedAllocationHeader)); - } - - RecordCuda *const record = - alloc_ptr ? static_cast(head.m_record) : nullptr; - - if (!alloc_ptr || record->m_alloc_ptr != head_cuda) { - Kokkos::Impl::throw_runtime_exception( - std::string("Kokkos::Impl::SharedAllocationRecord< Kokkos::CudaSpace , " - "void >::get_record ERROR")); - } - - return record; -} - -SharedAllocationRecord *SharedAllocationRecord< - Kokkos::CudaUVMSpace, void>::get_record(void *alloc_ptr) { - using Header = SharedAllocationHeader; - using RecordCuda = SharedAllocationRecord; - - Header *const h = - alloc_ptr ? reinterpret_cast
(alloc_ptr) - 1 : nullptr; - - if (!alloc_ptr || h->m_record->m_alloc_ptr != h) { - Kokkos::Impl::throw_runtime_exception( - std::string("Kokkos::Impl::SharedAllocationRecord< " - "Kokkos::CudaUVMSpace , void >::get_record ERROR")); - } - - return static_cast(h->m_record); -} - -SharedAllocationRecord - *SharedAllocationRecord::get_record( - void *alloc_ptr) { - using Header = SharedAllocationHeader; - using RecordCuda = SharedAllocationRecord; - - Header *const h = - alloc_ptr ? reinterpret_cast
(alloc_ptr) - 1 : nullptr; - - if (!alloc_ptr || h->m_record->m_alloc_ptr != h) { - Kokkos::Impl::throw_runtime_exception( - std::string("Kokkos::Impl::SharedAllocationRecord< " - "Kokkos::CudaHostPinnedSpace , void >::get_record ERROR")); - } - - return static_cast(h->m_record); -} - -// end SharedAllocationRecord::get_record() }}}1 -//============================================================================== - -//============================================================================== -// {{{1 - -// Iterate records to print orphaned memory ... -void SharedAllocationRecord::print_records( - std::ostream &s, const Kokkos::CudaSpace &, bool detail) { - (void)s; - (void)detail; -#ifdef KOKKOS_ENABLE_DEBUG - SharedAllocationRecord *r = &s_root_record; - - char buffer[256]; - - SharedAllocationHeader head; - - if (detail) { - do { - if (r->m_alloc_ptr) { - Kokkos::Impl::DeepCopy( - &head, r->m_alloc_ptr, sizeof(SharedAllocationHeader)); - } else { - head.m_label[0] = 0; - } - - // Formatting dependent on sizeof(uintptr_t) - const char *format_string; - - if (sizeof(uintptr_t) == sizeof(unsigned long)) { - format_string = - "Cuda addr( 0x%.12lx ) list( 0x%.12lx 0x%.12lx ) extent[ 0x%.12lx " - "+ %.8ld ] count(%d) dealloc(0x%.12lx) %s\n"; - } else if (sizeof(uintptr_t) == sizeof(unsigned long long)) { - format_string = - "Cuda addr( 0x%.12llx ) list( 0x%.12llx 0x%.12llx ) extent[ " - "0x%.12llx + %.8ld ] count(%d) dealloc(0x%.12llx) %s\n"; - } - - snprintf(buffer, 256, format_string, reinterpret_cast(r), - reinterpret_cast(r->m_prev), - reinterpret_cast(r->m_next), - reinterpret_cast(r->m_alloc_ptr), r->m_alloc_size, - r->m_count, reinterpret_cast(r->m_dealloc), - head.m_label); - s << buffer; - r = r->m_next; - } while (r != &s_root_record); - } else { - do { - if (r->m_alloc_ptr) { - Kokkos::Impl::DeepCopy( - &head, r->m_alloc_ptr, sizeof(SharedAllocationHeader)); - - // Formatting dependent on sizeof(uintptr_t) - const char *format_string; - - if (sizeof(uintptr_t) == sizeof(unsigned long)) { - format_string = "Cuda [ 0x%.12lx + %ld ] %s\n"; - } else if (sizeof(uintptr_t) == sizeof(unsigned long long)) { - format_string = "Cuda [ 0x%.12llx + %ld ] %s\n"; - } - - snprintf(buffer, 256, format_string, - reinterpret_cast(r->data()), r->size(), - head.m_label); - } else { - snprintf(buffer, 256, "Cuda [ 0 + 0 ]\n"); - } - s << buffer; - r = r->m_next; - } while (r != &s_root_record); - } -#else - Kokkos::Impl::throw_runtime_exception( - "SharedAllocationHeader::print_records only works with " - "KOKKOS_ENABLE_DEBUG enabled"); -#endif -} - -void SharedAllocationRecord::print_records( - std::ostream &s, const Kokkos::CudaUVMSpace &, bool detail) { - (void)s; - (void)detail; -#ifdef KOKKOS_ENABLE_DEBUG - SharedAllocationRecord::print_host_accessible_records( - s, "CudaUVM", &s_root_record, detail); -#else - Kokkos::Impl::throw_runtime_exception( - "SharedAllocationHeader::print_records only works with " - "KOKKOS_ENABLE_DEBUG enabled"); -#endif -} - -void SharedAllocationRecord::print_records( - std::ostream &s, const Kokkos::CudaHostPinnedSpace &, bool detail) { - (void)s; - (void)detail; -#ifdef KOKKOS_ENABLE_DEBUG - SharedAllocationRecord::print_host_accessible_records( - s, "CudaHostPinned", &s_root_record, detail); -#else - Kokkos::Impl::throw_runtime_exception( - "SharedAllocationHeader::print_records only works with " - "KOKKOS_ENABLE_DEBUG enabled"); -#endif -} - -// end SharedAllocationRecord::print_records() }}}1 -//============================================================================== - void cuda_prefetch_pointer(const Cuda &space, const void *ptr, size_t bytes, bool to_device) { if ((ptr == nullptr) || (bytes == 0)) return; @@ -984,6 +600,29 @@ void cuda_prefetch_pointer(const Cuda &space, const void *ptr, size_t bytes, } // namespace Impl } // namespace Kokkos + +//============================================================================== +// {{{1 + +#include + +namespace Kokkos { +namespace Impl { + +// To avoid additional compilation cost for something that's (mostly?) not +// performance sensitive, we explicity instantiate these CRTP base classes here, +// where we have access to the associated *_timpl.hpp header files. +template class SharedAllocationRecordCommon; +template class HostInaccessibleSharedAllocationRecordCommon; +template class SharedAllocationRecordCommon; +template class SharedAllocationRecordCommon; + +} // end namespace Impl +} // end namespace Kokkos + +// end Explicit instantiations of CRTP Base classes }}}1 +//============================================================================== + #else void KOKKOS_CORE_SRC_CUDA_CUDASPACE_PREVENT_LINK_ERROR() {} #endif // KOKKOS_ENABLE_CUDA diff --git a/lib/kokkos/core/src/Cuda/Kokkos_Cuda_BlockSize_Deduction.hpp b/lib/kokkos/core/src/Cuda/Kokkos_Cuda_BlockSize_Deduction.hpp index 0d6d3bdb3a..0f4259072d 100644 --- a/lib/kokkos/core/src/Cuda/Kokkos_Cuda_BlockSize_Deduction.hpp +++ b/lib/kokkos/core/src/Cuda/Kokkos_Cuda_BlockSize_Deduction.hpp @@ -140,7 +140,7 @@ inline int cuda_deduce_block_size(bool early_termination, } } - if (early_termination && blocks_per_sm != 0) break; + if (early_termination && opt_block_size != 0) break; } return opt_block_size; @@ -222,7 +222,8 @@ inline size_t get_shmem_per_sm_prefer_l1(cudaDeviceProp const& properties) { case 52: case 61: return 96; case 70: - case 80: return 8; + case 80: + case 86: return 8; case 75: return 32; default: Kokkos::Impl::throw_runtime_exception( diff --git a/lib/kokkos/core/src/Cuda/Kokkos_Cuda_Half.hpp b/lib/kokkos/core/src/Cuda/Kokkos_Cuda_Half.hpp index a9a62380e5..ec9c434fe6 100644 --- a/lib/kokkos/core/src/Cuda/Kokkos_Cuda_Half.hpp +++ b/lib/kokkos/core/src/Cuda/Kokkos_Cuda_Half.hpp @@ -175,30 +175,42 @@ class half_t { return cast_from_half(*this); } + /** + * Conversion constructors. + * + * Support implicit conversions from impl_type, float, double -> half_t + * Mixed precision expressions require upcasting which is done in the + * "// Binary Arithmetic" operator overloads below. + * + * Support implicit conversions from integral types -> half_t. + * Expressions involving half_t with integral types require downcasting + * the integral types to half_t. Existing operator overloads can handle this + * with the addition of the below implicit conversion constructors. + */ KOKKOS_FUNCTION half_t(impl_type rhs) : val(rhs) {} KOKKOS_FUNCTION - explicit half_t(float rhs) : val(cast_to_half(rhs).val) {} + half_t(float rhs) : val(cast_to_half(rhs).val) {} + KOKKOS_FUNCTION + half_t(double rhs) : val(cast_to_half(rhs).val) {} KOKKOS_FUNCTION explicit half_t(bool rhs) : val(cast_to_half(rhs).val) {} KOKKOS_FUNCTION - explicit half_t(double rhs) : val(cast_to_half(rhs).val) {} + half_t(short rhs) : val(cast_to_half(rhs).val) {} KOKKOS_FUNCTION - explicit half_t(short rhs) : val(cast_to_half(rhs).val) {} + half_t(int rhs) : val(cast_to_half(rhs).val) {} KOKKOS_FUNCTION - explicit half_t(int rhs) : val(cast_to_half(rhs).val) {} + half_t(long rhs) : val(cast_to_half(rhs).val) {} KOKKOS_FUNCTION - explicit half_t(long rhs) : val(cast_to_half(rhs).val) {} + half_t(long long rhs) : val(cast_to_half(rhs).val) {} KOKKOS_FUNCTION - explicit half_t(long long rhs) : val(cast_to_half(rhs).val) {} + half_t(unsigned short rhs) : val(cast_to_half(rhs).val) {} KOKKOS_FUNCTION - explicit half_t(unsigned short rhs) : val(cast_to_half(rhs).val) {} + half_t(unsigned int rhs) : val(cast_to_half(rhs).val) {} KOKKOS_FUNCTION - explicit half_t(unsigned int rhs) : val(cast_to_half(rhs).val) {} + half_t(unsigned long rhs) : val(cast_to_half(rhs).val) {} KOKKOS_FUNCTION - explicit half_t(unsigned long rhs) : val(cast_to_half(rhs).val) {} - KOKKOS_FUNCTION - explicit half_t(unsigned long long rhs) : val(cast_to_half(rhs).val) {} + half_t(unsigned long long rhs) : val(cast_to_half(rhs).val) {} // Unary operators KOKKOS_FUNCTION @@ -243,7 +255,7 @@ class half_t { #else float tmp = __half2float(val); --tmp; - val = __float2half(tmp); + val = __float2half(tmp); #endif return *this; } @@ -276,88 +288,317 @@ class half_t { return *this; } + template + KOKKOS_FUNCTION void operator=(T rhs) volatile { + val = cast_to_half(rhs).val; + } + // Compound operators KOKKOS_FUNCTION half_t& operator+=(half_t rhs) { #ifdef __CUDA_ARCH__ val += rhs.val; #else - val = __float2half(__half2float(val) + __half2float(rhs.val)); + val = __float2half(__half2float(val) + __half2float(rhs.val)); #endif return *this; } + KOKKOS_FUNCTION + volatile half_t& operator+=(half_t rhs) volatile { +#ifdef __CUDA_ARCH__ + // Cuda 10 supports __half volatile stores but not volatile arithmetic + // operands. Cast away volatile-ness of val for arithmetic but not for store + // location. + val = const_cast(val) + rhs.val; +#else + // Use non-volatile val_ref to suppress: + // "warning: implicit dereference will not access object of type ‘volatile + // __half’ in statement" + auto val_ref = const_cast(val); + val_ref = __float2half(__half2float(const_cast(val)) + + __half2float(rhs.val)); +#endif + return *this; + } + + // Compund operators: upcast overloads for += + template + KOKKOS_FUNCTION std::enable_if_t< + std::is_same::value || std::is_same::value, T> friend + operator+=(T& lhs, half_t rhs) { + lhs += static_cast(rhs); + return lhs; + } + + KOKKOS_FUNCTION + half_t& operator+=(float rhs) { + float result = static_cast(val) + rhs; + val = static_cast(result); + return *this; + } + + KOKKOS_FUNCTION + half_t& operator+=(double rhs) { + double result = static_cast(val) + rhs; + val = static_cast(result); + return *this; + } + KOKKOS_FUNCTION half_t& operator-=(half_t rhs) { #ifdef __CUDA_ARCH__ val -= rhs.val; #else - val = __float2half(__half2float(val) - __half2float(rhs.val)); + val = __float2half(__half2float(val) - __half2float(rhs.val)); #endif return *this; } + KOKKOS_FUNCTION + volatile half_t& operator-=(half_t rhs) volatile { +#ifdef __CUDA_ARCH__ + // Cuda 10 supports __half volatile stores but not volatile arithmetic + // operands. Cast away volatile-ness of val for arithmetic but not for store + // location. + val = const_cast(val) - rhs.val; +#else + // Use non-volatile val_ref to suppress: + // "warning: implicit dereference will not access object of type ‘volatile + // __half’ in statement" + auto val_ref = const_cast(val); + val_ref = __float2half(__half2float(const_cast(val)) - + __half2float(rhs.val)); +#endif + return *this; + } + + // Compund operators: upcast overloads for -= + template + KOKKOS_FUNCTION std::enable_if_t< + std::is_same::value || std::is_same::value, T> friend + operator-=(T& lhs, half_t rhs) { + lhs -= static_cast(rhs); + return lhs; + } + + KOKKOS_FUNCTION + half_t& operator-=(float rhs) { + float result = static_cast(val) - rhs; + val = static_cast(result); + return *this; + } + + KOKKOS_FUNCTION + half_t& operator-=(double rhs) { + double result = static_cast(val) - rhs; + val = static_cast(result); + return *this; + } + KOKKOS_FUNCTION half_t& operator*=(half_t rhs) { #ifdef __CUDA_ARCH__ val *= rhs.val; #else - val = __float2half(__half2float(val) * __half2float(rhs.val)); + val = __float2half(__half2float(val) * __half2float(rhs.val)); #endif return *this; } + KOKKOS_FUNCTION + volatile half_t& operator*=(half_t rhs) volatile { +#ifdef __CUDA_ARCH__ + // Cuda 10 supports __half volatile stores but not volatile arithmetic + // operands. Cast away volatile-ness of val for arithmetic but not for store + // location. + val = const_cast(val) * rhs.val; +#else + // Use non-volatile val_ref to suppress: + // "warning: implicit dereference will not access object of type ‘volatile + // __half’ in statement" + auto val_ref = const_cast(val); + val_ref = __float2half(__half2float(const_cast(val)) * + __half2float(rhs.val)); +#endif + return *this; + } + + // Compund operators: upcast overloads for *= + template + KOKKOS_FUNCTION std::enable_if_t< + std::is_same::value || std::is_same::value, T> friend + operator*=(T& lhs, half_t rhs) { + lhs *= static_cast(rhs); + return lhs; + } + + KOKKOS_FUNCTION + half_t& operator*=(float rhs) { + float result = static_cast(val) * rhs; + val = static_cast(result); + return *this; + } + + KOKKOS_FUNCTION + half_t& operator*=(double rhs) { + double result = static_cast(val) * rhs; + val = static_cast(result); + return *this; + } + KOKKOS_FUNCTION half_t& operator/=(half_t rhs) { #ifdef __CUDA_ARCH__ val /= rhs.val; #else - val = __float2half(__half2float(val) / __half2float(rhs.val)); + val = __float2half(__half2float(val) / __half2float(rhs.val)); #endif return *this; } + KOKKOS_FUNCTION + volatile half_t& operator/=(half_t rhs) volatile { +#ifdef __CUDA_ARCH__ + // Cuda 10 supports __half volatile stores but not volatile arithmetic + // operands. Cast away volatile-ness of val for arithmetic but not for store + // location. + val = const_cast(val) / rhs.val; +#else + // Use non-volatile val_ref to suppress: + // "warning: implicit dereference will not access object of type ‘volatile + // __half’ in statement" + auto val_ref = const_cast(val); + val_ref = __float2half(__half2float(const_cast(val)) / + __half2float(rhs.val)); +#endif + return *this; + } + + // Compund operators: upcast overloads for /= + template + KOKKOS_FUNCTION std::enable_if_t< + std::is_same::value || std::is_same::value, T> friend + operator/=(T& lhs, half_t rhs) { + lhs /= static_cast(rhs); + return lhs; + } + + KOKKOS_FUNCTION + half_t& operator/=(float rhs) { + float result = static_cast(val) / rhs; + val = static_cast(result); + return *this; + } + + KOKKOS_FUNCTION + half_t& operator/=(double rhs) { + double result = static_cast(val) / rhs; + val = static_cast(result); + return *this; + } + // Binary Arithmetic KOKKOS_FUNCTION half_t friend operator+(half_t lhs, half_t rhs) { #ifdef __CUDA_ARCH__ lhs.val += rhs.val; #else - lhs.val = __float2half(__half2float(lhs.val) + __half2float(rhs.val)); + lhs.val = __float2half(__half2float(lhs.val) + __half2float(rhs.val)); #endif return lhs; } + // Binary Arithmetic upcast operators for + + template + KOKKOS_FUNCTION std::enable_if_t< + std::is_same::value || std::is_same::value, T> friend + operator+(half_t lhs, T rhs) { + return T(lhs) + rhs; + } + + template + KOKKOS_FUNCTION std::enable_if_t< + std::is_same::value || std::is_same::value, T> friend + operator+(T lhs, half_t rhs) { + return lhs + T(rhs); + } + KOKKOS_FUNCTION half_t friend operator-(half_t lhs, half_t rhs) { #ifdef __CUDA_ARCH__ lhs.val -= rhs.val; #else - lhs.val = __float2half(__half2float(lhs.val) - __half2float(rhs.val)); + lhs.val = __float2half(__half2float(lhs.val) - __half2float(rhs.val)); #endif return lhs; } + // Binary Arithmetic upcast operators for - + template + KOKKOS_FUNCTION std::enable_if_t< + std::is_same::value || std::is_same::value, T> friend + operator-(half_t lhs, T rhs) { + return T(lhs) - rhs; + } + + template + KOKKOS_FUNCTION std::enable_if_t< + std::is_same::value || std::is_same::value, T> friend + operator-(T lhs, half_t rhs) { + return lhs - T(rhs); + } + KOKKOS_FUNCTION half_t friend operator*(half_t lhs, half_t rhs) { #ifdef __CUDA_ARCH__ lhs.val *= rhs.val; #else - lhs.val = __float2half(__half2float(lhs.val) * __half2float(rhs.val)); + lhs.val = __float2half(__half2float(lhs.val) * __half2float(rhs.val)); #endif return lhs; } + // Binary Arithmetic upcast operators for * + template + KOKKOS_FUNCTION std::enable_if_t< + std::is_same::value || std::is_same::value, T> friend + operator*(half_t lhs, T rhs) { + return T(lhs) * rhs; + } + + template + KOKKOS_FUNCTION std::enable_if_t< + std::is_same::value || std::is_same::value, T> friend + operator*(T lhs, half_t rhs) { + return lhs * T(rhs); + } + KOKKOS_FUNCTION half_t friend operator/(half_t lhs, half_t rhs) { #ifdef __CUDA_ARCH__ lhs.val /= rhs.val; #else - lhs.val = __float2half(__half2float(lhs.val) / __half2float(rhs.val)); + lhs.val = __float2half(__half2float(lhs.val) / __half2float(rhs.val)); #endif return lhs; } + // Binary Arithmetic upcast operators for / + template + KOKKOS_FUNCTION std::enable_if_t< + std::is_same::value || std::is_same::value, T> friend + operator/(half_t lhs, T rhs) { + return T(lhs) / rhs; + } + + template + KOKKOS_FUNCTION std::enable_if_t< + std::is_same::value || std::is_same::value, T> friend + operator/(T lhs, half_t rhs) { + return lhs / T(rhs); + } + // Logical operators KOKKOS_FUNCTION bool operator!() const { diff --git a/lib/kokkos/core/src/Cuda/Kokkos_Cuda_Instance.cpp b/lib/kokkos/core/src/Cuda/Kokkos_Cuda_Instance.cpp index b8e8163458..016cb6cdcb 100644 --- a/lib/kokkos/core/src/Cuda/Kokkos_Cuda_Instance.cpp +++ b/lib/kokkos/core/src/Cuda/Kokkos_Cuda_Instance.cpp @@ -54,6 +54,7 @@ #include #include #include +#include #include #include @@ -248,11 +249,11 @@ void CudaInternal::print_configuration(std::ostream &s) const { const CudaInternalDevices &dev_info = CudaInternalDevices::singleton(); #if defined(KOKKOS_ENABLE_CUDA) - s << "macro KOKKOS_ENABLE_CUDA : defined" << std::endl; + s << "macro KOKKOS_ENABLE_CUDA : defined\n"; #endif #if defined(CUDA_VERSION) s << "macro CUDA_VERSION = " << CUDA_VERSION << " = version " - << CUDA_VERSION / 1000 << "." << (CUDA_VERSION % 1000) / 10 << std::endl; + << CUDA_VERSION / 1000 << "." << (CUDA_VERSION % 1000) / 10 << '\n'; #endif for (int i = 0; i < dev_info.m_cudaDevCount; ++i) { @@ -274,7 +275,6 @@ CudaInternal::~CudaInternal() { m_scratchConcurrentBitset) { std::cerr << "Kokkos::Cuda ERROR: Failed to call Kokkos::Cuda::finalize()" << std::endl; - std::cerr.flush(); } m_cudaDev = -1; @@ -358,8 +358,7 @@ void CudaInternal::initialize(int cuda_device_id, cudaStream_t stream) { if (m_cudaArch == 0) { std::stringstream ss; - ss << "Kokkos::Cuda::initialize ERROR: likely mismatch of architecture" - << std::endl; + ss << "Kokkos::Cuda::initialize ERROR: likely mismatch of architecture\n"; std::string msg = ss.str(); Kokkos::abort(msg.c_str()); } @@ -373,7 +372,7 @@ void CudaInternal::initialize(int cuda_device_id, cudaStream_t stream) { "compute capability " << compiled_major << "." << compiled_minor << " on device with compute capability " << cudaProp.major << "." - << cudaProp.minor << " is not supported by CUDA!" << std::endl; + << cudaProp.minor << " is not supported by CUDA!\n"; std::string msg = ss.str(); Kokkos::abort(msg.c_str()); } @@ -458,7 +457,7 @@ void CudaInternal::initialize(int cuda_device_id, cudaStream_t stream) { Kokkos::Impl::SharedAllocationRecord; Record *const r = - Record::allocate(Kokkos::CudaSpace(), "InternalScratchBitset", + Record::allocate(Kokkos::CudaSpace(), "Kokkos::InternalScratchBitset", sizeof(uint32_t) * buffer_bound); Record::increment(r); @@ -492,17 +491,11 @@ void CudaInternal::initialize(int cuda_device_id, cudaStream_t stream) { #ifdef KOKKOS_ENABLE_CUDA_UVM if (Kokkos::show_warnings() && !cuda_launch_blocking()) { - std::cerr << "Kokkos::Cuda::initialize WARNING: Cuda is allocating into " - "UVMSpace by default" - << std::endl; - std::cerr << " without setting " - "CUDA_LAUNCH_BLOCKING=1." - << std::endl; - std::cerr << " The code must call " - "Cuda().fence() after each kernel" - << std::endl; - std::cerr << " or will likely crash when " - "accessing data on the host." + std::cerr << R"warning( +Kokkos::Cuda::initialize WARNING: Cuda is allocating into UVMSpace by default + without setting CUDA_LAUNCH_BLOCKING=1. + The code must call Cuda().fence() after each kernel + or will likely crash when accessing data on the host.)warning" << std::endl; } @@ -520,19 +513,13 @@ void CudaInternal::initialize(int cuda_device_id, cudaStream_t stream) { if (Kokkos::show_warnings() && (!visible_devices_one && !force_device_alloc)) { - std::cerr << "Kokkos::Cuda::initialize WARNING: Cuda is allocating into " - "UVMSpace by default" + std::cerr << R"warning( +Kokkos::Cuda::initialize WARNING: Cuda is allocating into UVMSpace by default + without setting CUDA_MANAGED_FORCE_DEVICE_ALLOC=1 or + setting CUDA_VISIBLE_DEVICES. + This could on multi GPU systems lead to severe performance" + penalties.)warning" << std::endl; - std::cerr << " without setting " - "CUDA_MANAGED_FORCE_DEVICE_ALLOC=1 or " - << std::endl; - std::cerr - << " setting CUDA_VISIBLE_DEVICES." - << std::endl; - std::cerr << " This could on multi GPU " - "systems lead to severe performance" - << std::endl; - std::cerr << " penalties." << std::endl; } #endif @@ -575,7 +562,7 @@ Cuda::size_type *CudaInternal::scratch_flags(const Cuda::size_type size) const { if (m_scratchFlags) Record::decrement(Record::get_record(m_scratchFlags)); Record *const r = - Record::allocate(Kokkos::CudaSpace(), "InternalScratchFlags", + Record::allocate(Kokkos::CudaSpace(), "Kokkos::InternalScratchFlags", (sizeof(ScratchGrain) * m_scratchFlagsCount)); Record::increment(r); @@ -600,7 +587,7 @@ Cuda::size_type *CudaInternal::scratch_space(const Cuda::size_type size) const { if (m_scratchSpace) Record::decrement(Record::get_record(m_scratchSpace)); Record *const r = - Record::allocate(Kokkos::CudaSpace(), "InternalScratchSpace", + Record::allocate(Kokkos::CudaSpace(), "Kokkos::InternalScratchSpace", (sizeof(ScratchGrain) * m_scratchSpaceCount)); Record::increment(r); @@ -624,7 +611,7 @@ Cuda::size_type *CudaInternal::scratch_unified( Record::decrement(Record::get_record(m_scratchUnified)); Record *const r = Record::allocate( - Kokkos::CudaHostPinnedSpace(), "InternalScratchUnified", + Kokkos::CudaHostPinnedSpace(), "Kokkos::InternalScratchUnified", (sizeof(ScratchGrain) * m_scratchUnifiedCount)); Record::increment(r); @@ -646,8 +633,9 @@ Cuda::size_type *CudaInternal::scratch_functor( if (m_scratchFunctor) Record::decrement(Record::get_record(m_scratchFunctor)); - Record *const r = Record::allocate( - Kokkos::CudaSpace(), "InternalScratchFunctor", m_scratchFunctorSize); + Record *const r = + Record::allocate(Kokkos::CudaSpace(), "Kokkos::InternalScratchFunctor", + m_scratchFunctorSize); Record::increment(r); @@ -662,7 +650,7 @@ void *CudaInternal::resize_team_scratch_space(std::int64_t bytes, if (m_team_scratch_current_size == 0) { m_team_scratch_current_size = bytes; m_team_scratch_ptr = Kokkos::kokkos_malloc( - "CudaSpace::ScratchMemory", m_team_scratch_current_size); + "Kokkos::CudaSpace::TeamScratchMemory", m_team_scratch_current_size); } if ((bytes > m_team_scratch_current_size) || ((bytes < m_team_scratch_current_size) && (force_shrink))) { @@ -676,6 +664,9 @@ void *CudaInternal::resize_team_scratch_space(std::int64_t bytes, //---------------------------------------------------------------------------- void CudaInternal::finalize() { + // skip if finalize() has already been called + if (was_finalized) return; + was_finalized = true; if (nullptr != m_scratchSpace || nullptr != m_scratchFlags) { // Only finalize this if we're the singleton @@ -719,6 +710,11 @@ void CudaInternal::finalize() { if (this == &singleton()) { cudaFreeHost(constantMemHostStaging); cudaEventDestroy(constantMemReusable); + auto &deep_copy_space = + Kokkos::Impl::cuda_get_deep_copy_space(/*initialize*/ false); + if (deep_copy_space) + deep_copy_space->impl_internal_space_instance()->finalize(); + cudaStreamDestroy(cuda_get_deep_copy_stream()); } } @@ -821,62 +817,23 @@ Cuda::size_type Cuda::device_arch() { void Cuda::impl_finalize() { Impl::CudaInternal::singleton().finalize(); } Cuda::Cuda() - : m_space_instance(&Impl::CudaInternal::singleton()), m_counter(nullptr) { + : m_space_instance(&Impl::CudaInternal::singleton(), + [](Impl::CudaInternal *) {}) { Impl::CudaInternal::singleton().verify_is_initialized( "Cuda instance constructor"); } Cuda::Cuda(cudaStream_t stream) - : m_space_instance(new Impl::CudaInternal), m_counter(new int(1)) { + : m_space_instance(new Impl::CudaInternal, [](Impl::CudaInternal *ptr) { + ptr->finalize(); + delete ptr; + }) { Impl::CudaInternal::singleton().verify_is_initialized( "Cuda instance constructor"); m_space_instance->initialize(Impl::CudaInternal::singleton().m_cudaDev, stream); } -KOKKOS_FUNCTION Cuda::Cuda(Cuda &&other) noexcept { - m_space_instance = other.m_space_instance; - other.m_space_instance = nullptr; - m_counter = other.m_counter; - other.m_counter = nullptr; -} - -KOKKOS_FUNCTION Cuda::Cuda(const Cuda &other) - : m_space_instance(other.m_space_instance), m_counter(other.m_counter) { -#ifndef KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_CUDA - if (m_counter) Kokkos::atomic_add(m_counter, 1); -#endif -} - -KOKKOS_FUNCTION Cuda &Cuda::operator=(Cuda &&other) noexcept { - m_space_instance = other.m_space_instance; - other.m_space_instance = nullptr; - m_counter = other.m_counter; - other.m_counter = nullptr; - return *this; -} - -KOKKOS_FUNCTION Cuda &Cuda::operator=(const Cuda &other) { - m_space_instance = other.m_space_instance; - m_counter = other.m_counter; -#ifndef KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_CUDA - if (m_counter) Kokkos::atomic_add(m_counter, 1); -#endif - return *this; -} - -KOKKOS_FUNCTION Cuda::~Cuda() noexcept { -#ifndef KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_CUDA - if (m_counter == nullptr) return; - int const count = Kokkos::atomic_fetch_sub(m_counter, 1); - if (count == 1) { - delete m_counter; - m_space_instance->finalize(); - delete m_space_instance; - } -#endif -} - void Cuda::print_configuration(std::ostream &s, const bool) { Impl::CudaInternal::singleton().print_configuration(s); } @@ -924,54 +881,53 @@ void CudaSpaceInitializer::fence() { Kokkos::Cuda::impl_static_fence(); } void CudaSpaceInitializer::print_configuration(std::ostream &msg, const bool detail) { - msg << "Device Execution Space:" << std::endl; - msg << " KOKKOS_ENABLE_CUDA: "; - msg << "yes" << std::endl; + msg << "Device Execution Space:\n"; + msg << " KOKKOS_ENABLE_CUDA: yes\n"; - msg << "Cuda Atomics:" << std::endl; + msg << "Cuda Atomics:\n"; msg << " KOKKOS_ENABLE_CUDA_ATOMICS: "; #ifdef KOKKOS_ENABLE_CUDA_ATOMICS - msg << "yes" << std::endl; + msg << "yes\n"; #else - msg << "no" << std::endl; + msg << "no\n"; #endif - msg << "Cuda Options:" << std::endl; + msg << "Cuda Options:\n"; msg << " KOKKOS_ENABLE_CUDA_LAMBDA: "; #ifdef KOKKOS_ENABLE_CUDA_LAMBDA - msg << "yes" << std::endl; + msg << "yes\n"; #else - msg << "no" << std::endl; + msg << "no\n"; #endif msg << " KOKKOS_ENABLE_CUDA_LDG_INTRINSIC: "; #ifdef KOKKOS_ENABLE_CUDA_LDG_INTRINSIC - msg << "yes" << std::endl; + msg << "yes\n"; #else - msg << "no" << std::endl; + msg << "no\n"; #endif msg << " KOKKOS_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE: "; #ifdef KOKKOS_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE - msg << "yes" << std::endl; + msg << "yes\n"; #else - msg << "no" << std::endl; + msg << "no\n"; #endif msg << " KOKKOS_ENABLE_CUDA_UVM: "; #ifdef KOKKOS_ENABLE_CUDA_UVM - msg << "yes" << std::endl; + msg << "yes\n"; #else - msg << "no" << std::endl; + msg << "no\n"; #endif msg << " KOKKOS_ENABLE_CUSPARSE: "; #ifdef KOKKOS_ENABLE_CUSPARSE - msg << "yes" << std::endl; + msg << "yes\n"; #else - msg << "no" << std::endl; + msg << "no\n"; #endif msg << " KOKKOS_ENABLE_CXX11_DISPATCH_LAMBDA: "; #ifdef KOKKOS_ENABLE_CXX11_DISPATCH_LAMBDA - msg << "yes" << std::endl; + msg << "yes\n"; #else - msg << "no" << std::endl; + msg << "no\n"; #endif msg << "\nCuda Runtime Configuration:" << std::endl; diff --git a/lib/kokkos/core/src/Cuda/Kokkos_Cuda_Instance.hpp b/lib/kokkos/core/src/Cuda/Kokkos_Cuda_Instance.hpp index 13773d70c5..aaec2c2926 100644 --- a/lib/kokkos/core/src/Cuda/Kokkos_Cuda_Instance.hpp +++ b/lib/kokkos/core/src/Cuda/Kokkos_Cuda_Instance.hpp @@ -17,30 +17,24 @@ namespace Kokkos { namespace Impl { struct CudaTraits { - enum : CudaSpace::size_type { WarpSize = 32 /* 0x0020 */ }; - enum : CudaSpace::size_type { - WarpIndexMask = 0x001f /* Mask for warpindex */ - }; - enum : CudaSpace::size_type { - WarpIndexShift = 5 /* WarpSize == 1 << WarpShift */ - }; + static constexpr CudaSpace::size_type WarpSize = 32 /* 0x0020 */; + static constexpr CudaSpace::size_type WarpIndexMask = + 0x001f; /* Mask for warpindex */ + static constexpr CudaSpace::size_type WarpIndexShift = + 5; /* WarpSize == 1 << WarpShift */ - enum : CudaSpace::size_type { - ConstantMemoryUsage = 0x008000 /* 32k bytes */ - }; - enum : CudaSpace::size_type { - ConstantMemoryCache = 0x002000 /* 8k bytes */ - }; - enum : CudaSpace::size_type { - KernelArgumentLimit = 0x001000 /* 4k bytes */ - }; - enum : CudaSpace::size_type { - MaxHierarchicalParallelism = 1024 /* team_size * vector_length */ - }; + static constexpr CudaSpace::size_type ConstantMemoryUsage = + 0x008000; /* 32k bytes */ + static constexpr CudaSpace::size_type ConstantMemoryCache = + 0x002000; /* 8k bytes */ + static constexpr CudaSpace::size_type KernelArgumentLimit = + 0x001000; /* 4k bytes */ + static constexpr CudaSpace::size_type MaxHierarchicalParallelism = + 1024; /* team_size * vector_length */ using ConstantGlobalBufferType = unsigned long[ConstantMemoryUsage / sizeof(unsigned long)]; - enum { ConstantMemoryUseThreshold = 0x000200 /* 512 bytes */ }; + static constexpr int ConstantMemoryUseThreshold = 0x000200 /* 512 bytes */; KOKKOS_INLINE_FUNCTION static CudaSpace::size_type warp_count( CudaSpace::size_type i) { diff --git a/lib/kokkos/core/src/Cuda/Kokkos_Cuda_KernelLaunch.hpp b/lib/kokkos/core/src/Cuda/Kokkos_Cuda_KernelLaunch.hpp index 39404e0bf3..d892a893b3 100644 --- a/lib/kokkos/core/src/Cuda/Kokkos_Cuda_KernelLaunch.hpp +++ b/lib/kokkos/core/src/Cuda/Kokkos_Cuda_KernelLaunch.hpp @@ -158,6 +158,9 @@ inline void check_shmem_request(CudaInternal const* cuda_instance, int shmem) { } } +// This function needs to be template on DriverType and LaunchBounds +// so that the static bool is unique for each type combo +// KernelFuncPtr does not necessarily contain that type information. template inline void configure_shmem_preference(KernelFuncPtr const& func, bool prefer_shmem) { @@ -355,8 +358,7 @@ struct CudaParallelLaunchKernelInvoker< if (!Impl::is_empty_launch(grid, block)) { Impl::check_shmem_request(cuda_instance, shmem); - Impl::configure_shmem_preference( + Impl::configure_shmem_preference( base_t::get_kernel_func(), prefer_shmem); void const* args[] = {&driver}; @@ -449,8 +451,7 @@ struct CudaParallelLaunchKernelInvoker< if (!Impl::is_empty_launch(grid, block)) { Impl::check_shmem_request(cuda_instance, shmem); - Impl::configure_shmem_preference( + Impl::configure_shmem_preference( base_t::get_kernel_func(), prefer_shmem); auto* driver_ptr = Impl::allocate_driver_storage_for_kernel(driver); @@ -627,9 +628,8 @@ struct CudaParallelLaunchImpl< get_cuda_func_attributes(), block, shmem, prefer_shmem); Impl::configure_shmem_preference< - DriverType, Kokkos::LaunchBounds, - decltype(base_t::get_kernel_func())>(base_t::get_kernel_func(), - prefer_shmem); + DriverType, Kokkos::LaunchBounds>( + base_t::get_kernel_func(), prefer_shmem); KOKKOS_ENSURE_CUDA_LOCK_ARRAYS_ON_DEVICE(); diff --git a/lib/kokkos/core/src/Cuda/Kokkos_Cuda_MDRangePolicy.hpp b/lib/kokkos/core/src/Cuda/Kokkos_Cuda_MDRangePolicy.hpp new file mode 100644 index 0000000000..12b7f70a97 --- /dev/null +++ b/lib/kokkos/core/src/Cuda/Kokkos_Cuda_MDRangePolicy.hpp @@ -0,0 +1,37 @@ +#ifndef KOKKOS_CUDA_MDRANGEPOLICY_HPP_ +#define KOKKOS_CUDA_MDRANGEPOLICY_HPP_ + +#include + +namespace Kokkos { + +template <> +struct default_outer_direction { + using type = Iterate; + static constexpr Iterate value = Iterate::Left; +}; + +template <> +struct default_inner_direction { + using type = Iterate; + static constexpr Iterate value = Iterate::Left; +}; + +namespace Impl { + +// Settings for MDRangePolicy +template <> +inline TileSizeProperties get_tile_size_properties( + const Kokkos::Cuda& space) { + TileSizeProperties properties; + properties.max_threads = + space.impl_internal_space_instance()->m_maxThreadsPerSM; + properties.default_largest_tile_size = 16; + properties.default_tile_size = 2; + properties.max_total_tile_size = 512; + return properties; +} + +} // Namespace Impl +} // Namespace Kokkos +#endif diff --git a/lib/kokkos/core/src/Cuda/Kokkos_Cuda_Parallel.hpp b/lib/kokkos/core/src/Cuda/Kokkos_Cuda_Parallel.hpp index 131d180980..2834e6f3de 100644 --- a/lib/kokkos/core/src/Cuda/Kokkos_Cuda_Parallel.hpp +++ b/lib/kokkos/core/src/Cuda/Kokkos_Cuda_Parallel.hpp @@ -60,6 +60,7 @@ #include #include #include +#include #include #include @@ -67,6 +68,7 @@ #include #include +#include //---------------------------------------------------------------------------- //---------------------------------------------------------------------------- @@ -474,7 +476,7 @@ class ParallelFor, Kokkos::Cuda> { Policy const& get_policy() const { return m_policy; } - inline __device__ void operator()(void) const { + inline __device__ void operator()() const { const Member work_stride = blockDim.y * gridDim.x; const Member work_end = m_policy.end(); @@ -537,9 +539,23 @@ class ParallelFor, Kokkos::Cuda> { const Policy m_rp; public: + template + static int max_tile_size_product(const Policy& pol, const Functor&) { + cudaFuncAttributes attr = + CudaParallelLaunch::get_cuda_func_attributes(); + auto const& prop = pol.space().cuda_device_prop(); + // Limits due to registers/SM, MDRange doesn't have + // shared memory constraints + int const regs_per_sm = prop.regsPerMultiprocessor; + int const regs_per_thread = attr.numRegs; + int const max_threads_per_sm = regs_per_sm / regs_per_thread; + return std::min( + max_threads_per_sm, + static_cast(Kokkos::Impl::CudaTraits::MaxHierarchicalParallelism)); + } Policy const& get_policy() const { return m_rp; } - - inline __device__ void operator()(void) const { + inline __device__ void operator()() const { Kokkos::Impl::DeviceIterateTile(m_rp, m_functor) .exec_range(); @@ -689,7 +705,7 @@ class ParallelFor, public: Policy const& get_policy() const { return m_policy; } - __device__ inline void operator()(void) const { + __device__ inline void operator()() const { // Iterate this block through the league int64_t threadid = 0; if (m_scratch_size[1] > 0) { @@ -1248,8 +1264,21 @@ class ParallelReduce, ReducerType, using DummySHMEMReductionType = int; public: + template + static int max_tile_size_product(const Policy& pol, const Functor&) { + cudaFuncAttributes attr = + CudaParallelLaunch::get_cuda_func_attributes(); + auto const& prop = pol.space().cuda_device_prop(); + // Limits due do registers/SM + int const regs_per_sm = prop.regsPerMultiprocessor; + int const regs_per_thread = attr.numRegs; + int const max_threads_per_sm = regs_per_sm / regs_per_thread; + return std::min( + max_threads_per_sm, + static_cast(Kokkos::Impl::CudaTraits::MaxHierarchicalParallelism)); + } Policy const& get_policy() const { return m_policy; } - inline __device__ void exec_range(reference_type update) const { Kokkos::Impl::Reduce::DeviceIterateTile, ReducerType, .exec_range(); } - inline __device__ void operator()(void) const { + inline __device__ void operator()() const { /* run(Kokkos::Impl::if_c::select(1,1.0) ); } @@ -2074,7 +2103,7 @@ class ParallelScan, Kokkos::Cuda> { //---------------------------------------- - __device__ inline void initial(void) const { + __device__ inline void initial() const { const integral_nonzero_constant word_count(ValueTraits::value_size(m_functor) / sizeof(size_type)); @@ -2110,7 +2139,7 @@ class ParallelScan, Kokkos::Cuda> { //---------------------------------------- - __device__ inline void final(void) const { + __device__ inline void final() const { const integral_nonzero_constant word_count(ValueTraits::value_size(m_functor) / sizeof(size_type)); @@ -2195,7 +2224,7 @@ class ParallelScan, Kokkos::Cuda> { //---------------------------------------- - __device__ inline void operator()(void) const { + __device__ inline void operator()() const { #ifdef KOKKOS_IMPL_DEBUG_CUDA_SERIAL_EXECUTION if (m_run_serial) { typename ValueTraits::value_type value; @@ -2364,7 +2393,7 @@ class ParallelScanWithTotal, //---------------------------------------- - __device__ inline void initial(void) const { + __device__ inline void initial() const { const integral_nonzero_constant word_count(ValueTraits::value_size(m_functor) / sizeof(size_type)); @@ -2400,7 +2429,7 @@ class ParallelScanWithTotal, //---------------------------------------- - __device__ inline void final(void) const { + __device__ inline void final() const { const integral_nonzero_constant word_count(ValueTraits::value_size(m_functor) / sizeof(size_type)); @@ -2487,7 +2516,7 @@ class ParallelScanWithTotal, //---------------------------------------- - __device__ inline void operator()(void) const { + __device__ inline void operator()() const { #ifdef KOKKOS_IMPL_DEBUG_CUDA_SERIAL_EXECUTION if (m_run_serial) { typename ValueTraits::value_type value; diff --git a/lib/kokkos/core/src/Cuda/Kokkos_Cuda_Team.hpp b/lib/kokkos/core/src/Cuda/Kokkos_Cuda_Team.hpp index 4b472f5d4f..e780639015 100644 --- a/lib/kokkos/core/src/Cuda/Kokkos_Cuda_Team.hpp +++ b/lib/kokkos/core/src/Cuda/Kokkos_Cuda_Team.hpp @@ -661,13 +661,14 @@ KOKKOS_INLINE_FUNCTION thread, count); } -template -KOKKOS_INLINE_FUNCTION - Impl::ThreadVectorRangeBoundariesStruct - ThreadVectorRange(const Impl::CudaTeamMember& thread, iType arg_begin, - iType arg_end) { +template +KOKKOS_INLINE_FUNCTION Impl::ThreadVectorRangeBoundariesStruct< + typename std::common_type::type, Impl::CudaTeamMember> +ThreadVectorRange(const Impl::CudaTeamMember& thread, iType1 arg_begin, + iType2 arg_end) { + using iType = typename std::common_type::type; return Impl::ThreadVectorRangeBoundariesStruct( - thread, arg_begin, arg_end); + thread, iType(arg_begin), iType(arg_end)); } KOKKOS_INLINE_FUNCTION @@ -983,7 +984,7 @@ KOKKOS_INLINE_FUNCTION void parallel_scan( //---------------------------------------------------------------------------- -/** \brief Intra-thread vector parallel exclusive prefix sum. +/** \brief Intra-thread vector parallel scan with reducer. * * Executes closure(iType i, ValueType & val, bool final) for each i=[0..N) * @@ -991,25 +992,25 @@ KOKKOS_INLINE_FUNCTION void parallel_scan( * thread and a scan operation is performed. * The last call to closure has final == true. */ -template -KOKKOS_INLINE_FUNCTION void parallel_scan( - const Impl::ThreadVectorRangeBoundariesStruct& - loop_boundaries, - const Closure& closure) { +template +KOKKOS_INLINE_FUNCTION + typename std::enable_if::value>::type + parallel_scan(const Impl::ThreadVectorRangeBoundariesStruct< + iType, Impl::CudaTeamMember>& loop_boundaries, + const Closure& closure, const ReducerType& reducer) { (void)loop_boundaries; (void)closure; + (void)reducer; #ifdef __CUDA_ARCH__ - // Extract value_type from closure - - using value_type = typename Kokkos::Impl::FunctorAnalysis< - Kokkos::Impl::FunctorPatternInterface::SCAN, void, Closure>::value_type; + using value_type = typename ReducerType::value_type; + value_type accum; + reducer.init(accum); + const value_type identity = accum; // Loop through boundaries by vector-length chunks // must scan at each iteration - value_type accum = 0; - // All thread "lanes" must loop the same number of times. // Determine an loop end for all thread "lanes." // Requires: @@ -1026,44 +1027,68 @@ KOKKOS_INLINE_FUNCTION void parallel_scan( const int end = loop_boundaries.end + (rem ? blockDim.x - rem : 0); for (int i = threadIdx.x; i < end; i += blockDim.x) { - value_type val = 0; + value_type val = identity; - // First acquire per-lane contributions: - if (i < loop_boundaries.end) closure(i, val, false); + // First acquire per-lane contributions. + // This sets i's val to i-1's contribution + // to make the latter in_place_shfl_up an + // exclusive scan -- the final accumulation + // of i's val will be included in the second + // closure call later. + if (i < loop_boundaries.end && threadIdx.x > 0) closure(i - 1, val, false); - value_type sval = val; - - // Bottom up inclusive scan in triangular pattern + // Bottom up exclusive scan in triangular pattern // where each CUDA thread is the root of a reduction tree // from the zeroth "lane" to itself. // [t] += [t-1] if t >= 1 // [t] += [t-2] if t >= 2 // [t] += [t-4] if t >= 4 // ... - + // This differs from the non-reducer overload, where an inclusive scan was + // implemented, because in general the binary operator cannot be inverted + // and we would not be able to remove the inclusive contribution by + // inversion. for (int j = 1; j < (int)blockDim.x; j <<= 1) { - value_type tmp = 0; - Impl::in_place_shfl_up(tmp, sval, j, blockDim.x, active_mask); + value_type tmp = identity; + Impl::in_place_shfl_up(tmp, val, j, blockDim.x, active_mask); if (j <= (int)threadIdx.x) { - sval += tmp; + reducer.join(val, tmp); } } - // Include accumulation and remove value for exclusive scan: - val = accum + sval - val; + // Include accumulation + reducer.join(val, accum); - // Provide exclusive scan value: + // Update i's contribution into the val + // and add it to accum for next round if (i < loop_boundaries.end) closure(i, val, true); - - // Accumulate the last value in the inclusive scan: - Impl::in_place_shfl(sval, sval, mask, blockDim.x, active_mask); - - accum += sval; + Impl::in_place_shfl(accum, val, mask, blockDim.x, active_mask); } #endif } +//---------------------------------------------------------------------------- + +/** \brief Intra-thread vector parallel exclusive prefix sum. + * + * Executes closure(iType i, ValueType & val, bool final) for each i=[0..N) + * + * The range [0..N) is mapped to all vector lanes in the + * thread and a scan operation is performed. + * The last call to closure has final == true. + */ +template +KOKKOS_INLINE_FUNCTION void parallel_scan( + const Impl::ThreadVectorRangeBoundariesStruct& + loop_boundaries, + const Closure& closure) { + using value_type = typename Kokkos::Impl::FunctorAnalysis< + Kokkos::Impl::FunctorPatternInterface::SCAN, void, Closure>::value_type; + value_type dummy; + parallel_scan(loop_boundaries, closure, Kokkos::Sum(dummy)); +} + } // namespace Kokkos namespace Kokkos { diff --git a/lib/kokkos/core/src/Cuda/Kokkos_Cuda_View.hpp b/lib/kokkos/core/src/Cuda/Kokkos_Cuda_View.hpp index f24abb377d..c55956ede9 100644 --- a/lib/kokkos/core/src/Cuda/Kokkos_Cuda_View.hpp +++ b/lib/kokkos/core/src/Cuda/Kokkos_Cuda_View.hpp @@ -139,7 +139,7 @@ struct CudaLDGFetch { template KOKKOS_INLINE_FUNCTION ValueType operator[](const iType& i) const { -#ifdef __CUDA_ARCH__ +#if defined(__CUDA_ARCH__) && (350 <= _CUDA_ARCH__) AliasType v = __ldg(reinterpret_cast(&m_ptr[i])); return *(reinterpret_cast(&v)); #else diff --git a/lib/kokkos/core/src/Cuda/Kokkos_Cuda_WorkGraphPolicy.hpp b/lib/kokkos/core/src/Cuda/Kokkos_Cuda_WorkGraphPolicy.hpp index 05876a9f02..fc52e41514 100644 --- a/lib/kokkos/core/src/Cuda/Kokkos_Cuda_WorkGraphPolicy.hpp +++ b/lib/kokkos/core/src/Cuda/Kokkos_Cuda_WorkGraphPolicy.hpp @@ -46,6 +46,7 @@ #define KOKKOS_CUDA_WORKGRAPHPOLICY_HPP #include +#include namespace Kokkos { namespace Impl { diff --git a/lib/kokkos/core/src/HIP/Kokkos_HIP_BlockSize_Deduction.hpp b/lib/kokkos/core/src/HIP/Kokkos_HIP_BlockSize_Deduction.hpp index 89135b6c45..9278d1bdc9 100644 --- a/lib/kokkos/core/src/HIP/Kokkos_HIP_BlockSize_Deduction.hpp +++ b/lib/kokkos/core/src/HIP/Kokkos_HIP_BlockSize_Deduction.hpp @@ -75,17 +75,6 @@ void hipOccupancy(int *numBlocks, int blockSize, int sharedmem) { hipOccupancy( numBlocks, blockSize, sharedmem); } -template -struct HIPGetMaxBlockSize; - -template -int hip_get_max_block_size(typename DriverType::functor_type const &f, - size_t const vector_length, - size_t const shmem_extra_block, - size_t const shmem_extra_thread) { - return HIPGetMaxBlockSize::get_block_size( - f, vector_length, shmem_extra_block, shmem_extra_thread); -} template int hip_internal_get_block_size(const F &condition_check, @@ -131,10 +120,6 @@ int hip_internal_get_block_size(const F &condition_check, int opt_block_size = (blocks_per_sm >= min_blocks_per_sm) ? block_size : min_blocks_per_sm; int opt_threads_per_sm = threads_per_sm; - // printf("BlockSizeMax: %i Shmem: %i %i %i %i Regs: %i %i Blocks: %i %i - // Achieved: %i %i Opt: %i %i\n",block_size, - // shmem_per_sm,max_shmem_per_block,functor_shmem,total_shmem, - // regs_per_sm,regs_per_wavefront,max_blocks_shmem,max_blocks_regs,blocks_per_sm,threads_per_sm,opt_block_size,opt_threads_per_sm); block_size -= HIPTraits::WarpSize; while (condition_check(blocks_per_sm) && (block_size >= HIPTraits::WarpSize)) { @@ -160,10 +145,6 @@ int hip_internal_get_block_size(const F &condition_check, opt_threads_per_sm = threads_per_sm; } } - // printf("BlockSizeMax: %i Shmem: %i %i %i %i Regs: %i %i Blocks: %i %i - // Achieved: %i %i Opt: %i %i\n",block_size, - // shmem_per_sm,max_shmem_per_block,functor_shmem,total_shmem, - // regs_per_sm,regs_per_wavefront,max_blocks_shmem,max_blocks_regs,blocks_per_sm,threads_per_sm,opt_block_size,opt_threads_per_sm); block_size -= HIPTraits::WarpSize; } return opt_block_size; @@ -178,62 +159,6 @@ int hip_get_max_block_size(const HIPInternal *hip_instance, [](int x) { return x == 0; }, hip_instance, attr, f, vector_length, shmem_block, shmem_thread); } -template -struct HIPGetMaxBlockSize { - static int get_block_size(typename DriverType::functor_type const &f, - size_t const vector_length, - size_t const shmem_extra_block, - size_t const shmem_extra_thread) { - int numBlocks = 0; - int blockSize = LaunchBounds::maxTperB == 0 ? 1024 : LaunchBounds::maxTperB; - int sharedmem = - shmem_extra_block + shmem_extra_thread * (blockSize / vector_length) + - ::Kokkos::Impl::FunctorTeamShmemSize< - typename DriverType::functor_type>::value(f, blockSize / - vector_length); - - hipOccupancy(&numBlocks, blockSize, sharedmem); - - if (numBlocks > 0) return blockSize; - while (blockSize > HIPTraits::WarpSize && numBlocks == 0) { - blockSize /= 2; - sharedmem = - shmem_extra_block + shmem_extra_thread * (blockSize / vector_length) + - ::Kokkos::Impl::FunctorTeamShmemSize< - typename DriverType::functor_type>::value(f, blockSize / - vector_length); - - hipOccupancy(&numBlocks, blockSize, sharedmem); - } - int blockSizeUpperBound = blockSize * 2; - while (blockSize < blockSizeUpperBound && numBlocks > 0) { - blockSize += HIPTraits::WarpSize; - sharedmem = - shmem_extra_block + shmem_extra_thread * (blockSize / vector_length) + - ::Kokkos::Impl::FunctorTeamShmemSize< - typename DriverType::functor_type>::value(f, blockSize / - vector_length); - - hipOccupancy(&numBlocks, blockSize, sharedmem); - } - return blockSize - HIPTraits::WarpSize; - } -}; - -template -struct HIPGetOptBlockSize; - -template -int hip_get_opt_block_size(typename DriverType::functor_type const &f, - size_t const vector_length, - size_t const shmem_extra_block, - size_t const shmem_extra_thread) { - return HIPGetOptBlockSize< - DriverType, LaunchBounds, - (HIPTraits::ConstantMemoryUseThreshold < - sizeof(DriverType))>::get_block_size(f, vector_length, shmem_extra_block, - shmem_extra_thread); -} template int hip_get_opt_block_size(HIPInternal const *hip_instance, @@ -245,157 +170,6 @@ int hip_get_opt_block_size(HIPInternal const *hip_instance, shmem_block, shmem_thread); } -// FIXME_HIP the code is identical to the false struct except for -// hip_parallel_launch_constant_memory -template -struct HIPGetOptBlockSize, true> { - static int get_block_size(typename DriverType::functor_type const &f, - size_t const vector_length, - size_t const shmem_extra_block, - size_t const shmem_extra_thread) { - int blockSize = HIPTraits::WarpSize / 2; - int numBlocks; - int sharedmem; - int maxOccupancy = 0; - int bestBlockSize = 0; - - while (blockSize < HIPTraits::MaxThreadsPerBlock) { - blockSize *= 2; - - // calculate the occupancy with that optBlockSize and check whether its - // larger than the largest one found so far - sharedmem = - shmem_extra_block + shmem_extra_thread * (blockSize / vector_length) + - ::Kokkos::Impl::FunctorTeamShmemSize< - typename DriverType::functor_type>::value(f, blockSize / - vector_length); - hipOccupancy(&numBlocks, blockSize, sharedmem); - if (maxOccupancy < numBlocks * blockSize) { - maxOccupancy = numBlocks * blockSize; - bestBlockSize = blockSize; - } - } - return bestBlockSize; - } -}; - -template -struct HIPGetOptBlockSize, false> { - static int get_block_size(const typename DriverType::functor_type &f, - const size_t vector_length, - const size_t shmem_extra_block, - const size_t shmem_extra_thread) { - int blockSize = HIPTraits::WarpSize / 2; - int numBlocks; - int sharedmem; - int maxOccupancy = 0; - int bestBlockSize = 0; - - while (blockSize < HIPTraits::MaxThreadsPerBlock) { - blockSize *= 2; - sharedmem = - shmem_extra_block + shmem_extra_thread * (blockSize / vector_length) + - ::Kokkos::Impl::FunctorTeamShmemSize< - typename DriverType::functor_type>::value(f, blockSize / - vector_length); - - hipOccupancy(&numBlocks, blockSize, sharedmem); - - if (maxOccupancy < numBlocks * blockSize) { - maxOccupancy = numBlocks * blockSize; - bestBlockSize = blockSize; - } - } - return bestBlockSize; - } -}; - -// FIXME_HIP the code is identical to the false struct except for -// hip_parallel_launch_constant_memory -template -struct HIPGetOptBlockSize< - DriverType, Kokkos::LaunchBounds, - true> { - static int get_block_size(const typename DriverType::functor_type &f, - const size_t vector_length, - const size_t shmem_extra_block, - const size_t shmem_extra_thread) { - int blockSize = HIPTraits::WarpSize / 2; - int numBlocks; - int sharedmem; - int maxOccupancy = 0; - int bestBlockSize = 0; - int max_threads_per_block = - std::min(MaxThreadsPerBlock, - hip_internal_maximum_warp_count() * HIPTraits::WarpSize); - - while (blockSize < max_threads_per_block) { - blockSize *= 2; - - // calculate the occupancy with that optBlockSize and check whether its - // larger than the largest one found so far - sharedmem = - shmem_extra_block + shmem_extra_thread * (blockSize / vector_length) + - ::Kokkos::Impl::FunctorTeamShmemSize< - typename DriverType::functor_type>::value(f, blockSize / - vector_length); - hipOccupancy( - &numBlocks, blockSize, sharedmem); - if (numBlocks >= static_cast(MinBlocksPerSM) && - blockSize <= static_cast(MaxThreadsPerBlock)) { - if (maxOccupancy < numBlocks * blockSize) { - maxOccupancy = numBlocks * blockSize; - bestBlockSize = blockSize; - } - } - } - if (maxOccupancy > 0) return bestBlockSize; - return -1; - } -}; - -template -struct HIPGetOptBlockSize< - DriverType, Kokkos::LaunchBounds, - false> { - static int get_block_size(const typename DriverType::functor_type &f, - const size_t vector_length, - const size_t shmem_extra_block, - const size_t shmem_extra_thread) { - int blockSize = HIPTraits::WarpSize / 2; - int numBlocks; - int sharedmem; - int maxOccupancy = 0; - int bestBlockSize = 0; - int max_threads_per_block = - std::min(MaxThreadsPerBlock, - hip_internal_maximum_warp_count() * HIPTraits::WarpSize); - - while (blockSize < max_threads_per_block) { - blockSize *= 2; - sharedmem = - shmem_extra_block + shmem_extra_thread * (blockSize / vector_length) + - ::Kokkos::Impl::FunctorTeamShmemSize< - typename DriverType::functor_type>::value(f, blockSize / - vector_length); - - hipOccupancy( - &numBlocks, blockSize, sharedmem); - if (numBlocks >= int(MinBlocksPerSM) && - blockSize <= int(MaxThreadsPerBlock)) { - if (maxOccupancy < numBlocks * blockSize) { - maxOccupancy = numBlocks * blockSize; - bestBlockSize = blockSize; - } - } - } - if (maxOccupancy > 0) return bestBlockSize; - return -1; - } -}; - } // namespace Impl } // namespace Experimental } // namespace Kokkos diff --git a/lib/kokkos/core/src/HIP/Kokkos_HIP_Instance.cpp b/lib/kokkos/core/src/HIP/Kokkos_HIP_Instance.cpp index 45512038ac..18ef10e22c 100644 --- a/lib/kokkos/core/src/HIP/Kokkos_HIP_Instance.cpp +++ b/lib/kokkos/core/src/HIP/Kokkos_HIP_Instance.cpp @@ -164,6 +164,8 @@ HIPInternal &HIPInternal::singleton() { void HIPInternal::fence() const { HIP_SAFE_CALL(hipStreamSynchronize(m_stream)); + // can reset our cycle id now as well + m_cycleId = 0; } void HIPInternal::initialize(int hip_device_id, hipStream_t stream) { @@ -256,7 +258,7 @@ void HIPInternal::initialize(int hip_device_id, hipStream_t stream) { void>; Record *const r = Record::allocate(Kokkos::Experimental::HIPSpace(), - "InternalScratchBitset", + "Kokkos::InternalScratchBitset", sizeof(uint32_t) * buffer_bound); Record::increment(r); @@ -303,8 +305,10 @@ Kokkos::Experimental::HIP::size_type *HIPInternal::scratch_space( Kokkos::Impl::SharedAllocationRecord; - static Record *const r = Record::allocate( - Kokkos::Experimental::HIPSpace(), "InternalScratchSpace", + if (m_scratchSpace) Record::decrement(Record::get_record(m_scratchSpace)); + + Record *const r = Record::allocate( + Kokkos::Experimental::HIPSpace(), "Kokkos::InternalScratchSpace", (sizeScratchGrain * m_scratchSpaceCount)); Record::increment(r); @@ -325,8 +329,10 @@ Kokkos::Experimental::HIP::size_type *HIPInternal::scratch_flags( Kokkos::Impl::SharedAllocationRecord; + if (m_scratchFlags) Record::decrement(Record::get_record(m_scratchFlags)); + Record *const r = Record::allocate( - Kokkos::Experimental::HIPSpace(), "InternalScratchFlags", + Kokkos::Experimental::HIPSpace(), "Kokkos::InternalScratchFlags", (sizeScratchGrain * m_scratchFlagsCount)); Record::increment(r); @@ -345,7 +351,7 @@ void *HIPInternal::resize_team_scratch_space(std::int64_t bytes, if (m_team_scratch_current_size == 0) { m_team_scratch_current_size = bytes; m_team_scratch_ptr = Kokkos::kokkos_malloc( - "HIPSpace::ScratchMemory", m_team_scratch_current_size); + "Kokkos::HIPSpace::TeamScratchMemory", m_team_scratch_current_size); } if ((bytes > m_team_scratch_current_size) || ((bytes < m_team_scratch_current_size) && (force_shrink))) { @@ -388,6 +394,40 @@ void HIPInternal::finalize() { m_team_scratch_current_size = 0; m_team_scratch_ptr = nullptr; } + if (nullptr != d_driverWorkArray) { + HIP_SAFE_CALL(hipHostFree(d_driverWorkArray)); + d_driverWorkArray = nullptr; + } +} + +char *HIPInternal::get_next_driver(size_t driverTypeSize) const { + std::lock_guard const lock(m_mutexWorkArray); + if (d_driverWorkArray == nullptr) { + HIP_SAFE_CALL( + hipHostMalloc(&d_driverWorkArray, + m_maxDriverCycles * m_maxDriverTypeSize * sizeof(char), + hipHostMallocNonCoherent)); + } + if (driverTypeSize > m_maxDriverTypeSize) { + // fence handles the cycle id reset for us + fence(); + HIP_SAFE_CALL(hipHostFree(d_driverWorkArray)); + m_maxDriverTypeSize = driverTypeSize; + if (m_maxDriverTypeSize % 128 != 0) + m_maxDriverTypeSize = + m_maxDriverTypeSize + 128 - m_maxDriverTypeSize % 128; + HIP_SAFE_CALL( + hipHostMalloc(&d_driverWorkArray, + m_maxDriverCycles * m_maxDriverTypeSize * sizeof(char), + hipHostMallocNonCoherent)); + } else { + m_cycleId = (m_cycleId + 1) % m_maxDriverCycles; + if (m_cycleId == 0) { + // ensure any outstanding kernels are completed before we wrap around + fence(); + } + } + return &d_driverWorkArray[m_maxDriverTypeSize * m_cycleId]; } //---------------------------------------------------------------------------- diff --git a/lib/kokkos/core/src/HIP/Kokkos_HIP_Instance.hpp b/lib/kokkos/core/src/HIP/Kokkos_HIP_Instance.hpp index 07ec8625e6..f4f88628e3 100644 --- a/lib/kokkos/core/src/HIP/Kokkos_HIP_Instance.hpp +++ b/lib/kokkos/core/src/HIP/Kokkos_HIP_Instance.hpp @@ -49,6 +49,8 @@ #include +#include + namespace Kokkos { namespace Experimental { namespace Impl { @@ -83,33 +85,46 @@ class HIPInternal { public: using size_type = ::Kokkos::Experimental::HIP::size_type; - int m_hipDev; - int m_hipArch; - unsigned m_multiProcCount; - unsigned m_maxWarpCount; - unsigned m_maxBlock; - unsigned m_maxBlocksPerSM; - unsigned m_maxSharedWords; + int m_hipDev = -1; + int m_hipArch = -1; + unsigned m_multiProcCount = 0; + unsigned m_maxWarpCount = 0; + unsigned m_maxBlock = 0; + unsigned m_maxBlocksPerSM = 0; + unsigned m_maxSharedWords = 0; int m_regsPerSM; - int m_shmemPerSM; - int m_maxShmemPerBlock; - int m_maxThreadsPerSM; + int m_shmemPerSM = 0; + int m_maxShmemPerBlock = 0; + int m_maxThreadsPerSM = 0; + + // array of DriverTypes to be allocated in host-pinned memory for async + // kernel launches + mutable char *d_driverWorkArray = nullptr; + // number of kernel launches that can be in-flight w/o synchronization + const int m_maxDriverCycles = 100; + // max size of a DriverType [bytes] + mutable size_t m_maxDriverTypeSize = 1024 * 10; + // the current index in the driverWorkArray + mutable int m_cycleId = 0; + // mutex to access d_driverWorkArray + mutable std::mutex m_mutexWorkArray; // Scratch Spaces for Reductions - size_type m_scratchSpaceCount; - size_type m_scratchFlagsCount; + size_type m_scratchSpaceCount = 0; + size_type m_scratchFlagsCount = 0; - size_type *m_scratchSpace; - size_type *m_scratchFlags; + size_type *m_scratchSpace = nullptr; + size_type *m_scratchFlags = nullptr; uint32_t *m_scratchConcurrentBitset = nullptr; hipDeviceProp_t m_deviceProp; - hipStream_t m_stream; + hipStream_t m_stream = nullptr; // Team Scratch Level 1 Space - mutable int64_t m_team_scratch_current_size; - mutable void *m_team_scratch_ptr; + mutable int64_t m_team_scratch_current_size = 0; + mutable void *m_team_scratch_ptr = nullptr; + mutable std::mutex m_team_scratch_mutex; bool was_finalized = false; @@ -117,9 +132,7 @@ class HIPInternal { int verify_is_initialized(const char *const label) const; - int is_initialized() const { - return m_hipDev >= 0; - } // 0 != m_scratchSpace && 0 != m_scratchFlags ; } + int is_initialized() const { return m_hipDev >= 0; } void initialize(int hip_device_id, hipStream_t stream = nullptr); void finalize(); @@ -128,25 +141,12 @@ class HIPInternal { void fence() const; + // returns the next driver type pointer in our work array + char *get_next_driver(size_t driverTypeSize) const; + ~HIPInternal(); - HIPInternal() - : m_hipDev(-1), - m_hipArch(-1), - m_multiProcCount(0), - m_maxWarpCount(0), - m_maxBlock(0), - m_maxSharedWords(0), - m_shmemPerSM(0), - m_maxShmemPerBlock(0), - m_maxThreadsPerSM(0), - m_scratchSpaceCount(0), - m_scratchFlagsCount(0), - m_scratchSpace(nullptr), - m_scratchFlags(nullptr), - m_stream(nullptr), - m_team_scratch_current_size(0), - m_team_scratch_ptr(nullptr) {} + HIPInternal() = default; // Resizing of reduction related scratch spaces size_type *scratch_space(const size_type size); diff --git a/lib/kokkos/core/src/HIP/Kokkos_HIP_KernelLaunch.hpp b/lib/kokkos/core/src/HIP/Kokkos_HIP_KernelLaunch.hpp index 3e972c7346..f774423b37 100644 --- a/lib/kokkos/core/src/HIP/Kokkos_HIP_KernelLaunch.hpp +++ b/lib/kokkos/core/src/HIP/Kokkos_HIP_KernelLaunch.hpp @@ -49,9 +49,9 @@ #if defined(__HIPCC__) -#include #include #include +#include // Must use global variable on the device with HIP-Clang #ifdef __HIP__ @@ -127,19 +127,69 @@ struct HIPDispatchProperties { HIPLaunchMechanism launch_mechanism = l; }; -template , +template +struct HIPParallelLaunchKernelFunc; + +template +struct HIPParallelLaunchKernelFunc< + DriverType, Kokkos::LaunchBounds, + HIPLaunchMechanism::LocalMemory> { + static auto get_kernel_func() { + return hip_parallel_launch_local_memory; + } +}; + +template +struct HIPParallelLaunchKernelFunc, + HIPLaunchMechanism::LocalMemory> { + static auto get_kernel_func() { + return hip_parallel_launch_local_memory; + } +}; + +template +struct HIPParallelLaunchKernelInvoker; + +template +struct HIPParallelLaunchKernelInvoker + : HIPParallelLaunchKernelFunc { + using base_t = HIPParallelLaunchKernelFunc; + + static void invoke_kernel(DriverType const *driver, dim3 const &grid, + dim3 const &block, int shmem, + HIPInternal const *hip_instance) { + (base_t::get_kernel_func())<<m_stream>>>( + driver); + } +}; + +template , HIPLaunchMechanism LaunchMechanism = HIPLaunchMechanism::LocalMemory> struct HIPParallelLaunch; -template struct HIPParallelLaunch< DriverType, Kokkos::LaunchBounds, - HIPLaunchMechanism::LocalMemory> { - inline HIPParallelLaunch(const DriverType &driver, const dim3 &grid, - const dim3 &block, const int shmem, - const HIPInternal *hip_instance, - const bool /*prefer_shmem*/) { + HIPLaunchMechanism::LocalMemory> + : HIPParallelLaunchKernelInvoker< + DriverType, Kokkos::LaunchBounds, + HIPLaunchMechanism::LocalMemory> { + using base_t = HIPParallelLaunchKernelInvoker< + DriverType, Kokkos::LaunchBounds, + HIPLaunchMechanism::LocalMemory>; + + HIPParallelLaunch(const DriverType &driver, const dim3 &grid, + const dim3 &block, const int shmem, + const HIPInternal *hip_instance, + const bool /*prefer_shmem*/) { if ((grid.x != 0) && ((block.x * block.y * block.z) != 0)) { if (hip_instance->m_maxShmemPerBlock < shmem) { Kokkos::Impl::throw_runtime_exception( @@ -148,72 +198,16 @@ struct HIPParallelLaunch< KOKKOS_ENSURE_HIP_LOCK_ARRAYS_ON_DEVICE(); - // FIXME_HIP -- there is currently an error copying (some) structs - // by value to the device in HIP-Clang / VDI - // As a workaround, we can malloc the DriverType and explictly copy over. - // To remove once solved in HIP - DriverType *d_driver; - HIP_SAFE_CALL(hipMalloc(&d_driver, sizeof(DriverType))); - HIP_SAFE_CALL(hipMemcpyAsync(d_driver, &driver, sizeof(DriverType), - hipMemcpyHostToDevice, - hip_instance->m_stream)); - hip_parallel_launch_local_memory - <<m_stream>>>(d_driver); - -#if defined(KOKKOS_ENABLE_DEBUG_BOUNDS_CHECK) - HIP_SAFE_CALL(hipGetLastError()); - hip_instance->fence(); -#endif - HIP_SAFE_CALL(hipFree(d_driver)); - } - } - - static hipFuncAttributes get_hip_func_attributes() { - static hipFuncAttributes attr = []() { - hipFuncAttributes attr; - HIP_SAFE_CALL(hipFuncGetAttributes( - &attr, - reinterpret_cast( - hip_parallel_launch_local_memory))); - return attr; - }(); - return attr; - } -}; - -template -struct HIPParallelLaunch, - HIPLaunchMechanism::LocalMemory> { - inline HIPParallelLaunch(const DriverType &driver, const dim3 &grid, - const dim3 &block, const int shmem, - const HIPInternal *hip_instance, - const bool /*prefer_shmem*/) { - if ((grid.x != 0) && ((block.x * block.y * block.z) != 0)) { - if (hip_instance->m_maxShmemPerBlock < shmem) { - Kokkos::Impl::throw_runtime_exception(std::string( - "HIPParallelLaunch FAILED: shared memory request is too large")); - } - - KOKKOS_ENSURE_HIP_LOCK_ARRAYS_ON_DEVICE(); - // Invoke the driver function on the device - - // FIXME_HIP -- see note about struct copy by value above - DriverType *d_driver; - HIP_SAFE_CALL(hipMalloc(&d_driver, sizeof(DriverType))); - HIP_SAFE_CALL(hipMemcpyAsync(d_driver, &driver, sizeof(DriverType), - hipMemcpyHostToDevice, - hip_instance->m_stream)); - hip_parallel_launch_local_memory - <<m_stream>>>(d_driver); + DriverType *d_driver = reinterpret_cast( + hip_instance->get_next_driver(sizeof(DriverType))); + std::memcpy((void *)d_driver, (void *)&driver, sizeof(DriverType)); + base_t::invoke_kernel(d_driver, grid, block, shmem, hip_instance); #if defined(KOKKOS_ENABLE_DEBUG_BOUNDS_CHECK) HIP_SAFE_CALL(hipGetLastError()); hip_instance->fence(); #endif - HIP_SAFE_CALL(hipFree(d_driver)); } } @@ -221,8 +215,7 @@ struct HIPParallelLaunch, static hipFuncAttributes attr = []() { hipFuncAttributes attr; HIP_SAFE_CALL(hipFuncGetAttributes( - &attr, reinterpret_cast( - hip_parallel_launch_local_memory))); + &attr, reinterpret_cast(base_t::get_kernel_func()))); return attr; }(); return attr; diff --git a/lib/kokkos/core/src/HIP/Kokkos_HIP_MDRangePolicy.hpp b/lib/kokkos/core/src/HIP/Kokkos_HIP_MDRangePolicy.hpp new file mode 100644 index 0000000000..ce1aff9586 --- /dev/null +++ b/lib/kokkos/core/src/HIP/Kokkos_HIP_MDRangePolicy.hpp @@ -0,0 +1,37 @@ +#ifndef KOKKOS_HIP_MDRANGEPOLICY_HPP_ +#define KOKKOS_HIP_MDRANGEPOLICY_HPP_ + +#include + +namespace Kokkos { + +template <> +struct default_outer_direction { + using type = Iterate; + static constexpr Iterate value = Iterate::Left; +}; + +template <> +struct default_inner_direction { + using type = Iterate; + static constexpr Iterate value = Iterate::Left; +}; + +namespace Impl { + +// Settings for MDRangePolicy +template <> +inline TileSizeProperties get_tile_size_properties( + const Kokkos::Experimental::HIP& space) { + TileSizeProperties properties; + properties.max_threads = + space.impl_internal_space_instance()->m_maxThreadsPerSM; + properties.default_largest_tile_size = 16; + properties.default_tile_size = 4; + properties.max_total_tile_size = 1024; + return properties; +} + +} // Namespace Impl +} // Namespace Kokkos +#endif diff --git a/lib/kokkos/core/src/HIP/Kokkos_HIP_Parallel_MDRange.hpp b/lib/kokkos/core/src/HIP/Kokkos_HIP_Parallel_MDRange.hpp index 6b831ff7a3..35e7d6fb85 100644 --- a/lib/kokkos/core/src/HIP/Kokkos_HIP_Parallel_MDRange.hpp +++ b/lib/kokkos/core/src/HIP/Kokkos_HIP_Parallel_MDRange.hpp @@ -49,6 +49,7 @@ #include #include #include +#include #include namespace Kokkos { @@ -72,7 +73,7 @@ class ParallelFor, ParallelFor& operator=(ParallelFor const&) = delete; public: - inline __device__ void operator()(void) const { + inline __device__ void operator()() const { Kokkos::Impl::DeviceIterateTile(m_policy, m_functor) @@ -175,6 +176,25 @@ class ParallelFor, ParallelFor(FunctorType const& arg_functor, Policy const& arg_policy) : m_functor(arg_functor), m_policy(arg_policy) {} + + template + static int max_tile_size_product(const Policy& pol, const Functor&) { + using closure_type = + ParallelFor, + Kokkos::Experimental::HIP>; + hipFuncAttributes attr = Kokkos::Experimental::Impl::HIPParallelLaunch< + closure_type, LaunchBounds>::get_hip_func_attributes(); + auto const& prop = pol.space().hip_device_prop(); + // Limits due to registers/SM, MDRange doesn't have + // shared memory constraints + int const regs_per_sm = prop.regsPerMultiprocessor; + int const regs_per_thread = attr.numRegs; + int const max_threads_per_sm = regs_per_sm / regs_per_thread; + return std::min( + max_threads_per_sm, + static_cast( + Kokkos::Experimental::Impl::HIPTraits::MaxThreadsPerBlock)); + } }; // ParallelReduce @@ -231,7 +251,7 @@ class ParallelReduce, ReducerType, DeviceIteratePattern(m_policy, m_functor, update).exec_range(); } - inline __device__ void operator()(void) const { + inline __device__ void operator()() const { const integral_nonzero_constant word_count(ValueTraits::value_size( @@ -291,13 +311,19 @@ class ParallelReduce, ReducerType, ::Kokkos::Experimental::Impl::HIPTraits::MaxThreadsPerBlock; int shmem_size = ::Kokkos::Impl::hip_single_inter_block_reduce_scan_shmem< false, FunctorType, WorkTag>(f, n); + using closure_type = Impl::ParallelReduce; + hipFuncAttributes attr = ::Kokkos::Experimental::Impl::HIPParallelLaunch< + closure_type, LaunchBounds>::get_hip_func_attributes(); while ( (n && (m_policy.space().impl_internal_space_instance()->m_maxShmemPerBlock < shmem_size)) || - (n > static_cast( - ::Kokkos::Experimental::Impl::hip_get_max_block_size< - ParallelReduce, LaunchBounds>(f, 1, shmem_size, 0)))) { + (n > + static_cast( + ::Kokkos::Experimental::Impl::hip_get_max_block_size( + m_policy.space().impl_internal_space_instance(), attr, f, 1, + shmem_size, 0)))) { n >>= 1; shmem_size = ::Kokkos::Impl::hip_single_inter_block_reduce_scan_shmem< false, FunctorType, WorkTag>(f, n); @@ -391,6 +417,23 @@ class ParallelReduce, ReducerType, memory_space>::accessible), m_scratch_space(nullptr), m_scratch_flags(nullptr) {} + template + static int max_tile_size_product(const Policy& pol, const Functor&) { + using closure_type = + ParallelReduce, + ReducerType, Kokkos::Experimental::HIP>; + hipFuncAttributes attr = Kokkos::Experimental::Impl::HIPParallelLaunch< + closure_type, LaunchBounds>::get_hip_func_attributes(); + auto const& prop = pol.space().hip_device_prop(); + // Limits due do registers/SM + int const regs_per_sm = prop.regsPerMultiprocessor; + int const regs_per_thread = attr.numRegs; + int const max_threads_per_sm = regs_per_sm / regs_per_thread; + return std::min( + max_threads_per_sm, + static_cast( + Kokkos::Experimental::Impl::HIPTraits::MaxThreadsPerBlock)); + } }; } // namespace Impl } // namespace Kokkos diff --git a/lib/kokkos/core/src/HIP/Kokkos_HIP_Parallel_Range.hpp b/lib/kokkos/core/src/HIP/Kokkos_HIP_Parallel_Range.hpp index 5607f1c91a..7d2825eeb4 100644 --- a/lib/kokkos/core/src/HIP/Kokkos_HIP_Parallel_Range.hpp +++ b/lib/kokkos/core/src/HIP/Kokkos_HIP_Parallel_Range.hpp @@ -92,7 +92,7 @@ class ParallelFor, public: using functor_type = FunctorType; - inline __device__ void operator()(void) const { + inline __device__ void operator()() const { const Member work_stride = blockDim.y * gridDim.x; const Member work_end = m_policy.end(); @@ -174,11 +174,14 @@ class ParallelReduce, ReducerType, size_type* m_scratch_space = nullptr; size_type* m_scratch_flags = nullptr; - // FIXME_HIP_PERFORMANCE Need a rule to choose when to use shared memory and - // when to use shuffle +#if HIP_VERSION < 401 static bool constexpr UseShflReduction = ((sizeof(value_type) > 2 * sizeof(double)) && static_cast(ValueTraits::StaticValueSize)); +#else + static bool constexpr UseShflReduction = + static_cast(ValueTraits::StaticValueSize); +#endif private: struct ShflReductionTag {}; @@ -330,13 +333,19 @@ class ParallelReduce, ReducerType, int shmem_size = hip_single_inter_block_reduce_scan_shmem( f, n); + using closure_type = Impl::ParallelReduce; + hipFuncAttributes attr = ::Kokkos::Experimental::Impl::HIPParallelLaunch< + closure_type, LaunchBounds>::get_hip_func_attributes(); while ( (n && (m_policy.space().impl_internal_space_instance()->m_maxShmemPerBlock < shmem_size)) || - (n > static_cast( - Kokkos::Experimental::Impl::hip_get_max_block_size< - ParallelReduce, LaunchBounds>(f, 1, shmem_size, 0)))) { + (n > + static_cast( + ::Kokkos::Experimental::Impl::hip_get_max_block_size( + m_policy.space().impl_internal_space_instance(), attr, f, 1, + shmem_size, 0)))) { n >>= 1; shmem_size = hip_single_inter_block_reduce_scan_shmem( @@ -493,7 +502,7 @@ class ParallelScanHIPBase { //---------------------------------------- - __device__ inline void initial(void) const { + __device__ inline void initial() const { const integral_nonzero_constant word_count(ValueTraits::value_size(m_functor) / sizeof(size_type)); @@ -529,7 +538,7 @@ class ParallelScanHIPBase { //---------------------------------------- - __device__ inline void final(void) const { + __device__ inline void final() const { const integral_nonzero_constant word_count(ValueTraits::value_size(m_functor) / sizeof(size_type)); @@ -606,7 +615,7 @@ class ParallelScanHIPBase { public: //---------------------------------------- - __device__ inline void operator()(void) const { + __device__ inline void operator()() const { if (!m_final) { initial(); } else { diff --git a/lib/kokkos/core/src/HIP/Kokkos_HIP_Parallel_Team.hpp b/lib/kokkos/core/src/HIP/Kokkos_HIP_Parallel_Team.hpp index 5da83d289e..96c3ff2a75 100644 --- a/lib/kokkos/core/src/HIP/Kokkos_HIP_Parallel_Team.hpp +++ b/lib/kokkos/core/src/HIP/Kokkos_HIP_Parallel_Team.hpp @@ -433,6 +433,9 @@ class ParallelFor, int m_shmem_size; void* m_scratch_ptr[2]; int m_scratch_size[2]; + // Only let one ParallelFor/Reduce modify the team scratch memory. The + // constructor acquires the mutex which is released in the destructor. + std::unique_lock m_scratch_lock; template __device__ inline @@ -449,7 +452,7 @@ class ParallelFor, } public: - __device__ inline void operator()(void) const { + __device__ inline void operator()() const { // Iterate this block through the league int64_t threadid = 0; if (m_scratch_size[1] > 0) { @@ -513,7 +516,10 @@ class ParallelFor, m_policy(arg_policy), m_league_size(arg_policy.league_size()), m_team_size(arg_policy.team_size()), - m_vector_size(arg_policy.impl_vector_length()) { + m_vector_size(arg_policy.impl_vector_length()), + m_scratch_lock(m_policy.space() + .impl_internal_space_instance() + ->m_team_scratch_mutex) { hipFuncAttributes attr = ::Kokkos::Experimental::Impl::HIPParallelLaunch< ParallelFor, launch_bounds>::get_hip_func_attributes(); m_team_size = @@ -640,6 +646,9 @@ class ParallelReduce, const size_type m_league_size; int m_team_size; const size_type m_vector_size; + // Only let one ParallelFor/Reduce modify the team scratch memory. The + // constructor acquires the mutex which is released in the destructor. + std::unique_lock m_scratch_lock; template __device__ inline @@ -877,7 +886,10 @@ class ParallelReduce, m_scratch_ptr{nullptr, nullptr}, m_league_size(arg_policy.league_size()), m_team_size(arg_policy.team_size()), - m_vector_size(arg_policy.impl_vector_length()) { + m_vector_size(arg_policy.impl_vector_length()), + m_scratch_lock(m_policy.space() + .impl_internal_space_instance() + ->m_team_scratch_mutex) { hipFuncAttributes attr = Kokkos::Experimental::Impl::HIPParallelLaunch< ParallelReduce, launch_bounds>::get_hip_func_attributes(); m_team_size = @@ -976,7 +988,10 @@ class ParallelReduce, m_scratch_ptr{nullptr, nullptr}, m_league_size(arg_policy.league_size()), m_team_size(arg_policy.team_size()), - m_vector_size(arg_policy.impl_vector_length()) { + m_vector_size(arg_policy.impl_vector_length()), + m_scratch_lock(m_policy.space() + .impl_internal_space_instance() + ->m_team_scratch_mutex) { hipFuncAttributes attr = Kokkos::Experimental::Impl::HIPParallelLaunch< ParallelReduce, launch_bounds>::get_hip_func_attributes(); m_team_size = diff --git a/lib/kokkos/core/src/HIP/Kokkos_HIP_Space.cpp b/lib/kokkos/core/src/HIP/Kokkos_HIP_Space.cpp index 00cef28f82..15ca089d14 100644 --- a/lib/kokkos/core/src/HIP/Kokkos_HIP_Space.cpp +++ b/lib/kokkos/core/src/HIP/Kokkos_HIP_Space.cpp @@ -42,12 +42,6 @@ //@HEADER */ -#include -#include -#include -#include -#include -#include #include #include @@ -57,6 +51,13 @@ #include #include +#include +#include +#include +#include +#include +#include + /*--------------------------------------------------------------------------*/ /*--------------------------------------------------------------------------*/ namespace Kokkos { @@ -172,14 +173,14 @@ void DeepCopyAsyncHIP(void* dst, void const* src, size_t n) { namespace Kokkos { -void Experimental::HIPSpace::access_error() { +KOKKOS_DEPRECATED void Experimental::HIPSpace::access_error() { const std::string msg( "Kokkos::Experimental::HIPSpace::access_error attempt to execute " "Experimental::HIP function from non-HIP space"); Kokkos::Impl::throw_runtime_exception(msg); } -void Experimental::HIPSpace::access_error(const void* const) { +KOKKOS_DEPRECATED void Experimental::HIPSpace::access_error(const void* const) { const std::string msg( "Kokkos::Experimental::HIPSpace::access_error attempt to execute " "Experimental::HIP function from non-HIP space"); @@ -326,45 +327,6 @@ SharedAllocationRecord SharedAllocationRecord< Kokkos::Experimental::HIPHostPinnedSpace, void>::s_root_record; #endif -std::string SharedAllocationRecord::get_label() const { - SharedAllocationHeader header; - - Kokkos::Impl::DeepCopy( - &header, RecordBase::head(), sizeof(SharedAllocationHeader)); - - return std::string(header.m_label); -} - -std::string SharedAllocationRecord::get_label() const { - return std::string(RecordBase::head()->m_label); -} - -SharedAllocationRecord* -SharedAllocationRecord::allocate( - const Kokkos::Experimental::HIPSpace& arg_space, - const std::string& arg_label, const size_t arg_alloc_size) { - return new SharedAllocationRecord(arg_space, arg_label, arg_alloc_size); -} - -SharedAllocationRecord* -SharedAllocationRecord:: - allocate(const Kokkos::Experimental::HIPHostPinnedSpace& arg_space, - const std::string& arg_label, const size_t arg_alloc_size) { - return new SharedAllocationRecord(arg_space, arg_label, arg_alloc_size); -} - -void SharedAllocationRecord::deallocate( - SharedAllocationRecord* arg_rec) { - delete static_cast(arg_rec); -} - -void SharedAllocationRecord:: - deallocate(SharedAllocationRecord* arg_rec) { - delete static_cast(arg_rec); -} - SharedAllocationRecord::~SharedAllocationRecord() { const char* label = nullptr; @@ -393,7 +355,7 @@ SharedAllocationRecord:: const SharedAllocationRecord::function_type arg_dealloc) // Pass through allocated [ SharedAllocationHeader , user_memory ] // Pass through deallocation function - : SharedAllocationRecord( + : base_t( #ifdef KOKKOS_ENABLE_DEBUG &SharedAllocationRecord::s_root_record, @@ -405,13 +367,7 @@ SharedAllocationRecord:: SharedAllocationHeader header; - // Fill in the Header information - header.m_record = static_cast*>(this); - - strncpy(header.m_label, arg_label.c_str(), - SharedAllocationHeader::maximum_label_length); - // Set last element zero, in case c_str is too long - header.m_label[SharedAllocationHeader::maximum_label_length - 1] = (char)0; + this->base_t::_fill_host_accessible_header_info(header, arg_label); // Copy to device memory Kokkos::Impl::DeepCopy( @@ -425,7 +381,7 @@ SharedAllocationRecord:: const SharedAllocationRecord::function_type arg_dealloc) // Pass through allocated [ SharedAllocationHeader , user_memory ] // Pass through deallocation function - : SharedAllocationRecord( + : base_t( #ifdef KOKKOS_ENABLE_DEBUG &SharedAllocationRecord::s_root_record, @@ -435,223 +391,8 @@ SharedAllocationRecord:: sizeof(SharedAllocationHeader) + arg_alloc_size, arg_dealloc), m_space(arg_space) { // Fill in the Header information, directly accessible via host pinned memory - - RecordBase::m_alloc_ptr->m_record = this; - - strncpy(RecordBase::m_alloc_ptr->m_label, arg_label.c_str(), - SharedAllocationHeader::maximum_label_length); - // Set last element zero, in case c_str is too long - RecordBase::m_alloc_ptr - ->m_label[SharedAllocationHeader::maximum_label_length - 1] = (char)0; -} - -//---------------------------------------------------------------------------- - -void* SharedAllocationRecord:: - allocate_tracked(const Kokkos::Experimental::HIPSpace& arg_space, - const std::string& arg_alloc_label, - const size_t arg_alloc_size) { - if (!arg_alloc_size) return nullptr; - - SharedAllocationRecord* const r = - allocate(arg_space, arg_alloc_label, arg_alloc_size); - - RecordBase::increment(r); - - return r->data(); -} - -void SharedAllocationRecord::deallocate_tracked(void* const - arg_alloc_ptr) { - if (arg_alloc_ptr != nullptr) { - SharedAllocationRecord* const r = get_record(arg_alloc_ptr); - - RecordBase::decrement(r); - } -} - -void* SharedAllocationRecord:: - reallocate_tracked(void* const arg_alloc_ptr, const size_t arg_alloc_size) { - SharedAllocationRecord* const r_old = get_record(arg_alloc_ptr); - SharedAllocationRecord* const r_new = - allocate(r_old->m_space, r_old->get_label(), arg_alloc_size); - - Kokkos::Impl::DeepCopy( - r_new->data(), r_old->data(), std::min(r_old->size(), r_new->size())); - - RecordBase::increment(r_new); - RecordBase::decrement(r_old); - - return r_new->data(); -} - -void* SharedAllocationRecord:: - allocate_tracked(const Kokkos::Experimental::HIPHostPinnedSpace& arg_space, - const std::string& arg_alloc_label, - const size_t arg_alloc_size) { - if (!arg_alloc_size) return nullptr; - - SharedAllocationRecord* const r = - allocate(arg_space, arg_alloc_label, arg_alloc_size); - - RecordBase::increment(r); - - return r->data(); -} - -void SharedAllocationRecord::deallocate_tracked(void* const - arg_alloc_ptr) { - if (arg_alloc_ptr) { - SharedAllocationRecord* const r = get_record(arg_alloc_ptr); - - RecordBase::decrement(r); - } -} - -void* SharedAllocationRecord:: - reallocate_tracked(void* const arg_alloc_ptr, const size_t arg_alloc_size) { - SharedAllocationRecord* const r_old = get_record(arg_alloc_ptr); - SharedAllocationRecord* const r_new = - allocate(r_old->m_space, r_old->get_label(), arg_alloc_size); - - using HIPHostPinnedSpace = Kokkos::Experimental::HIPHostPinnedSpace; - Kokkos::Impl::DeepCopy( - r_new->data(), r_old->data(), std::min(r_old->size(), r_new->size())); - - RecordBase::increment(r_new); - RecordBase::decrement(r_old); - - return r_new->data(); -} - -//---------------------------------------------------------------------------- - -SharedAllocationRecord* -SharedAllocationRecord::get_record( - void* alloc_ptr) { - using Header = SharedAllocationHeader; - using RecordHIP = - SharedAllocationRecord; - - // Copy the header from the allocation - Header head; - - Header const* const head_hip = - alloc_ptr ? Header::get_header(alloc_ptr) : nullptr; - - if (alloc_ptr) { - Kokkos::Impl::DeepCopy( - &head, head_hip, sizeof(SharedAllocationHeader)); - } - - RecordHIP* const record = - alloc_ptr ? static_cast(head.m_record) : nullptr; - - if (!alloc_ptr || record->m_alloc_ptr != head_hip) { - Kokkos::Impl::throw_runtime_exception(std::string( - "Kokkos::Impl::SharedAllocationRecord< Kokkos::Experimental::HIPSpace " - ", void >::get_record ERROR")); - } - - return record; -} - -SharedAllocationRecord* -SharedAllocationRecord::get_record(void* alloc_ptr) { - using Header = SharedAllocationHeader; - using RecordHIP = - SharedAllocationRecord; - - Header* const h = - alloc_ptr ? reinterpret_cast(alloc_ptr) - 1 : nullptr; - - if (!alloc_ptr || h->m_record->m_alloc_ptr != h) { - Kokkos::Impl::throw_runtime_exception(std::string( - "Kokkos::Impl::SharedAllocationRecord< " - "Kokkos::Experimental::HIPHostPinnedSpace , void >::get_record ERROR")); - } - - return static_cast(h->m_record); -} - -// Iterate records to print orphaned memory ... -void SharedAllocationRecord:: - print_records(std::ostream& s, const Kokkos::Experimental::HIPSpace&, - bool detail) { -#ifdef KOKKOS_ENABLE_DEBUG - SharedAllocationRecord* r = &s_root_record; - - char buffer[256]; - - SharedAllocationHeader head; - - if (detail) { - do { - if (r->m_alloc_ptr) { - Kokkos::Impl::DeepCopy( - &head, r->m_alloc_ptr, sizeof(SharedAllocationHeader)); - } else { - head.m_label[0] = 0; - } - - // Formatting dependent on sizeof(uintptr_t) - const char* format_string; - - if (sizeof(uintptr_t) == sizeof(unsigned long)) { - format_string = - "HIP addr( 0x%.12lx ) list( 0x%.12lx 0x%.12lx ) extent[ 0x%.12lx + " - "%.8ld ] count(%d) dealloc(0x%.12lx) %s\n"; - } else if (sizeof(uintptr_t) == sizeof(unsigned long long)) { - format_string = - "HIP addr( 0x%.12llx ) list( 0x%.12llx 0x%.12llx ) extent[ " - "0x%.12llx + %.8ld ] count(%d) dealloc(0x%.12llx) %s\n"; - } - - snprintf(buffer, 256, format_string, reinterpret_cast(r), - reinterpret_cast(r->m_prev), - reinterpret_cast(r->m_next), - reinterpret_cast(r->m_alloc_ptr), r->m_alloc_size, - r->m_count, reinterpret_cast(r->m_dealloc), - head.m_label); - s << buffer; - r = r->m_next; - } while (r != &s_root_record); - } else { - do { - if (r->m_alloc_ptr) { - Kokkos::Impl::DeepCopy( - &head, r->m_alloc_ptr, sizeof(SharedAllocationHeader)); - - // Formatting dependent on sizeof(uintptr_t) - const char* format_string; - - if (sizeof(uintptr_t) == sizeof(unsigned long)) { - format_string = "HIP [ 0x%.12lx + %ld ] %s\n"; - } else if (sizeof(uintptr_t) == sizeof(unsigned long long)) { - format_string = "HIP [ 0x%.12llx + %ld ] %s\n"; - } - - snprintf(buffer, 256, format_string, - reinterpret_cast(r->data()), r->size(), - head.m_label); - } else { - snprintf(buffer, 256, "HIP [ 0 + 0 ]\n"); - } - s << buffer; - r = r->m_next; - } while (r != &s_root_record); - } -#else - (void)s; - (void)detail; - throw_runtime_exception( - "Kokkos::Impl::SharedAllocationRecord::print_records" - " only works with KOKKOS_ENABLE_DEBUG enabled"); -#endif + this->base_t::_fill_host_accessible_header_info(*RecordBase::m_alloc_ptr, + arg_label); } } // namespace Impl @@ -680,63 +421,22 @@ void HIP::impl_initialize(const HIP::SelectDevice config) { void HIP::impl_finalize() { Impl::HIPInternal::singleton().finalize(); } HIP::HIP() - : m_space_instance(&Impl::HIPInternal::singleton()), m_counter(nullptr) { + : m_space_instance(&Impl::HIPInternal::singleton(), + [](Impl::HIPInternal*) {}) { Impl::HIPInternal::singleton().verify_is_initialized( "HIP instance constructor"); } HIP::HIP(hipStream_t const stream) - : m_space_instance(new Impl::HIPInternal), m_counter(new int(1)) { + : m_space_instance(new Impl::HIPInternal, [](Impl::HIPInternal* ptr) { + ptr->finalize(); + delete ptr; + }) { Impl::HIPInternal::singleton().verify_is_initialized( "HIP instance constructor"); m_space_instance->initialize(Impl::HIPInternal::singleton().m_hipDev, stream); } -KOKKOS_FUNCTION HIP::HIP(HIP&& other) noexcept { - m_space_instance = other.m_space_instance; - other.m_space_instance = nullptr; - m_counter = other.m_counter; - other.m_counter = nullptr; -} - -KOKKOS_FUNCTION HIP::HIP(HIP const& other) - : m_space_instance(other.m_space_instance), m_counter(other.m_counter) { -#ifndef KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HIP_GPU - if (m_counter) Kokkos::atomic_add(m_counter, 1); -#endif -} - -KOKKOS_FUNCTION HIP& HIP::operator=(HIP&& other) noexcept { - m_space_instance = other.m_space_instance; - other.m_space_instance = nullptr; - m_counter = other.m_counter; - other.m_counter = nullptr; - - return *this; -} - -KOKKOS_FUNCTION HIP& HIP::operator=(HIP const& other) { - m_space_instance = other.m_space_instance; - m_counter = other.m_counter; -#ifndef KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HIP_GPU - if (m_counter) Kokkos::atomic_add(m_counter, 1); -#endif - - return *this; -} - -KOKKOS_FUNCTION HIP::~HIP() noexcept { -#ifndef KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HIP_GPU - if (m_counter == nullptr) return; - int const count = Kokkos::atomic_fetch_sub(m_counter, 1); - if (count == 1) { - delete m_counter; - m_space_instance->finalize(); - delete m_space_instance; - } -#endif -} - void HIP::print_configuration(std::ostream& s, const bool) { Impl::HIPInternal::singleton().print_configuration(s); } @@ -810,3 +510,26 @@ void HIPSpaceInitializer::print_configuration(std::ostream& msg, } // namespace Impl } // namespace Kokkos + +//============================================================================== +// {{{1 + +#include + +namespace Kokkos { +namespace Impl { + +// To avoid additional compilation cost for something that's (mostly?) not +// performance sensitive, we explicity instantiate these CRTP base classes here, +// where we have access to the associated *_timpl.hpp header files. +template class HostInaccessibleSharedAllocationRecordCommon< + Kokkos::Experimental::HIPSpace>; +template class SharedAllocationRecordCommon; +template class SharedAllocationRecordCommon< + Kokkos::Experimental::HIPHostPinnedSpace>; + +} // end namespace Impl +} // end namespace Kokkos + +// end Explicit instantiations of CRTP Base classes }}}1 +//============================================================================== diff --git a/lib/kokkos/core/src/HIP/Kokkos_HIP_Team.hpp b/lib/kokkos/core/src/HIP/Kokkos_HIP_Team.hpp index 7571510c31..fe52886ced 100644 --- a/lib/kokkos/core/src/HIP/Kokkos_HIP_Team.hpp +++ b/lib/kokkos/core/src/HIP/Kokkos_HIP_Team.hpp @@ -644,13 +644,14 @@ KOKKOS_INLINE_FUNCTION thread, count); } -template -KOKKOS_INLINE_FUNCTION - Impl::ThreadVectorRangeBoundariesStruct - ThreadVectorRange(const Impl::HIPTeamMember& thread, iType arg_begin, - iType arg_end) { +template +KOKKOS_INLINE_FUNCTION Impl::ThreadVectorRangeBoundariesStruct< + typename std::common_type::type, Impl::HIPTeamMember> +ThreadVectorRange(const Impl::HIPTeamMember& thread, iType1 arg_begin, + iType2 arg_end) { + using iType = typename std::common_type::type; return Impl::ThreadVectorRangeBoundariesStruct( - thread, arg_begin, arg_end); + thread, iType(arg_begin), iType(arg_end)); } KOKKOS_INLINE_FUNCTION @@ -961,7 +962,7 @@ KOKKOS_INLINE_FUNCTION //---------------------------------------------------------------------------- -/** \brief Intra-thread vector parallel exclusive prefix sum. +/** \brief Intra-thread vector parallel scan with reducer. * * Executes closure(iType i, ValueType & val, bool final) for each i=[0..N) * @@ -969,22 +970,21 @@ KOKKOS_INLINE_FUNCTION * thread and a scan operation is performed. * The last call to closure has final == true. */ -template -KOKKOS_INLINE_FUNCTION void parallel_scan( - const Impl::ThreadVectorRangeBoundariesStruct& - loop_boundaries, - const Closure& closure) { +template +KOKKOS_INLINE_FUNCTION + typename std::enable_if::value>::type + parallel_scan(const Impl::ThreadVectorRangeBoundariesStruct< + iType, Impl::HIPTeamMember>& loop_boundaries, + const Closure& closure, const ReducerType& reducer) { #ifdef __HIP_DEVICE_COMPILE__ - // Extract value_type from closure - - using value_type = typename Kokkos::Impl::FunctorAnalysis< - Kokkos::Impl::FunctorPatternInterface::SCAN, void, Closure>::value_type; + using value_type = typename ReducerType::value_type; + value_type accum; + reducer.init(accum); + const value_type identity = accum; // Loop through boundaries by vector-length chunks // must scan at each iteration - value_type accum = 0; - // All thread "lanes" must loop the same number of times. // Determine an loop end for all thread "lanes." // Requires: @@ -997,47 +997,72 @@ KOKKOS_INLINE_FUNCTION void parallel_scan( const int end = loop_boundaries.end + (rem ? blockDim.x - rem : 0); for (int i = threadIdx.x; i < end; i += blockDim.x) { - value_type val = 0; + value_type val = identity; - // First acquire per-lane contributions: - if (i < loop_boundaries.end) closure(i, val, false); + // First acquire per-lane contributions. + // This sets i's val to i-1's contribution + // to make the latter in_place_shfl_up an + // exclusive scan -- the final accumulation + // of i's val will be included in the second + // closure call later. + if (i < loop_boundaries.end && threadIdx.x > 0) closure(i - 1, val, false); - value_type sval = val; - - // Bottom up inclusive scan in triangular pattern + // Bottom up exclusive scan in triangular pattern // where each HIP thread is the root of a reduction tree // from the zeroth "lane" to itself. // [t] += [t-1] if t >= 1 // [t] += [t-2] if t >= 2 // [t] += [t-4] if t >= 4 // ... - + // This differs from the non-reducer overload, where an inclusive scan was + // implemented, because in general the binary operator cannot be inverted + // and we would not be able to remove the inclusive contribution by + // inversion. for (int j = 1; j < static_cast(blockDim.x); j <<= 1) { - value_type tmp = 0; - ::Kokkos::Experimental::Impl::in_place_shfl_up(tmp, sval, j, blockDim.x); + value_type tmp = identity; + ::Kokkos::Experimental::Impl::in_place_shfl_up(tmp, val, j, blockDim.x); if (j <= static_cast(threadIdx.x)) { - sval += tmp; + reducer.join(val, tmp); } } - // Include accumulation and remove value for exclusive scan: - val = accum + sval - val; + // Include accumulation + reducer.join(val, accum); - // Provide exclusive scan value: + // Update i's contribution into the val + // and add it to accum for next round if (i < loop_boundaries.end) closure(i, val, true); - - // Accumulate the last value in the inclusive scan: - ::Kokkos::Experimental::Impl::in_place_shfl(sval, sval, blockDim.x - 1, + ::Kokkos::Experimental::Impl::in_place_shfl(accum, val, blockDim.x - 1, blockDim.x); - - accum += sval; } #else (void)loop_boundaries; (void)closure; + (void)reducer; #endif } +//---------------------------------------------------------------------------- + +/** \brief Intra-thread vector parallel exclusive prefix sum. + * + * Executes closure(iType i, ValueType & val, bool final) for each i=[0..N) + * + * The range [0..N) is mapped to all vector lanes in the + * thread and a scan operation is performed. + * The last call to closure has final == true. + */ +template +KOKKOS_INLINE_FUNCTION void parallel_scan( + const Impl::ThreadVectorRangeBoundariesStruct& + loop_boundaries, + const Closure& closure) { + using value_type = typename Kokkos::Impl::FunctorAnalysis< + Kokkos::Impl::FunctorPatternInterface::SCAN, void, Closure>::value_type; + value_type dummy; + parallel_scan(loop_boundaries, closure, Kokkos::Sum(dummy)); +} + } // namespace Kokkos namespace Kokkos { diff --git a/lib/kokkos/core/src/KokkosExp_MDRangePolicy.hpp b/lib/kokkos/core/src/KokkosExp_MDRangePolicy.hpp index 140376425c..b7d8e62f69 100644 --- a/lib/kokkos/core/src/KokkosExp_MDRangePolicy.hpp +++ b/lib/kokkos/core/src/KokkosExp_MDRangePolicy.hpp @@ -48,17 +48,11 @@ #include #include - +#include #include #include -#include #include -#if defined(KOKKOS_ENABLE_CUDA) || \ - (defined(__HIPCC__) && defined(KOKKOS_ENABLE_HIP)) -#include -#endif - namespace Kokkos { // ------------------------------------------------------------------ // @@ -74,22 +68,14 @@ enum class Iterate template struct default_outer_direction { - using type = Iterate; -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - static constexpr Iterate value = Iterate::Left; -#else + using type = Iterate; static constexpr Iterate value = Iterate::Right; -#endif }; template struct default_inner_direction { - using type = Iterate; -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - static constexpr Iterate value = Iterate::Left; -#else + using type = Iterate; static constexpr Iterate value = Iterate::Right; -#endif }; // Iteration Pattern @@ -179,6 +165,25 @@ constexpr NVCC_WONT_LET_ME_CALL_YOU_Array to_array_potentially_narrowing( } return a; } + +struct TileSizeProperties { + int max_threads; + int default_largest_tile_size; + int default_tile_size; + int max_total_tile_size; +}; + +template +TileSizeProperties get_tile_size_properties(const ExecutionSpace&) { + // Host settings + TileSizeProperties properties; + properties.max_threads = std::numeric_limits::max(); + properties.default_largest_tile_size = 0; + properties.default_tile_size = 2; + properties.max_total_tile_size = std::numeric_limits::max(); + return properties; +} + } // namespace Impl // multi-dimensional iteration pattern @@ -208,7 +213,7 @@ struct MDRangePolicy : public Kokkos::Impl::PolicyTraits { using launch_bounds = typename traits::launch_bounds; using member_type = typename range_policy::member_type; - enum { rank = static_cast(iteration_pattern::rank) }; + static constexpr int rank = iteration_pattern::rank; using index_type = typename traits::index_type; using array_index_type = std::int64_t; @@ -231,37 +236,20 @@ struct MDRangePolicy : public Kokkos::Impl::PolicyTraits { point_type m_tile_end = {}; index_type m_num_tiles = 1; index_type m_prod_tile_dims = 1; + bool m_tune_tile_size = false; - /* - // NDE enum impl definition alternative - replace static constexpr int ? - enum { outer_direction = static_cast ( - (iteration_pattern::outer_direction != Iterate::Default) - ? iteration_pattern::outer_direction - : default_outer_direction< typename traits::execution_space>::value ) }; - - enum { inner_direction = static_cast ( - iteration_pattern::inner_direction != Iterate::Default - ? iteration_pattern::inner_direction - : default_inner_direction< typename traits::execution_space>::value ) }; - - enum { Right = static_cast( Iterate::Right ) }; - enum { Left = static_cast( Iterate::Left ) }; - */ - // static constexpr int rank = iteration_pattern::rank; - - static constexpr int outer_direction = static_cast( + static constexpr auto outer_direction = (iteration_pattern::outer_direction != Iterate::Default) ? iteration_pattern::outer_direction - : default_outer_direction::value); + : default_outer_direction::value; - static constexpr int inner_direction = static_cast( + static constexpr auto inner_direction = iteration_pattern::inner_direction != Iterate::Default ? iteration_pattern::inner_direction - : default_inner_direction::value); + : default_inner_direction::value; - // Ugly ugly workaround intel 14 not handling scoped enum correctly - static constexpr int Right = static_cast(Iterate::Right); - static constexpr int Left = static_cast(Iterate::Left); + static constexpr auto Right = Iterate::Right; + static constexpr auto Left = Iterate::Left; KOKKOS_INLINE_FUNCTION const typename traits::execution_space& space() const { return m_space; @@ -320,7 +308,7 @@ struct MDRangePolicy : public Kokkos::Impl::PolicyTraits { point_type const& lower, point_type const& upper, tile_type const& tile = tile_type{}) : m_space(work_space), m_lower(lower), m_upper(upper), m_tile(tile) { - init(); + init_helper(Impl::get_tile_size_properties(work_space)); } template { m_tile(p.m_tile), m_tile_end(p.m_tile_end), m_num_tiles(p.m_num_tiles), - m_prod_tile_dims(p.m_prod_tile_dims) {} + m_prod_tile_dims(p.m_prod_tile_dims), + m_tune_tile_size(p.m_tune_tile_size) {} + + void impl_change_tile_size(const point_type& tile) { + m_tile = tile; + init_helper(Impl::get_tile_size_properties(m_space)); + } + bool impl_tune_tile_size() const { return m_tune_tile_size; } private: - void init() { - // Host - if (true -#if defined(KOKKOS_ENABLE_CUDA) - && !std::is_same::value -#endif -#if defined(KOKKOS_ENABLE_HIP) - && !std::is_same::value -#endif - ) { - index_type span; - for (int i = 0; i < rank; ++i) { - span = m_upper[i] - m_lower[i]; - if (m_tile[i] <= 0) { - if (((int)inner_direction == (int)Right && (i < rank - 1)) || - ((int)inner_direction == (int)Left && (i > 0))) { - m_tile[i] = 2; - } else { - m_tile[i] = (span == 0 ? 1 : span); - } - } - m_tile_end[i] = - static_cast((span + m_tile[i] - 1) / m_tile[i]); - m_num_tiles *= m_tile_end[i]; - m_prod_tile_dims *= m_tile[i]; - } + void init_helper(Impl::TileSizeProperties properties) { + m_prod_tile_dims = 1; + int increment = 1; + int rank_start = 0; + int rank_end = rank; + if (inner_direction == Iterate::Right) { + increment = -1; + rank_start = rank - 1; + rank_end = -1; } -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - else // Cuda or HIP - { - index_type span; - int increment = 1; - int rank_start = 0; - int rank_end = rank; - if ((int)inner_direction == (int)Right) { - increment = -1; - rank_start = rank - 1; - rank_end = -1; - } - bool is_cuda_exec_space = -#if defined(KOKKOS_ENABLE_CUDA) - std::is_same::value; -#else - false; -#endif - for (int i = rank_start; i != rank_end; i += increment) { - span = m_upper[i] - m_lower[i]; - if (m_tile[i] <= 0) { - // TODO: determine what is a good default tile size for Cuda and HIP - // may be rank dependent - if (((int)inner_direction == (int)Right && (i < rank - 1)) || - ((int)inner_direction == (int)Left && (i > 0))) { - if (m_prod_tile_dims < 256) { - m_tile[i] = (is_cuda_exec_space) ? 2 : 4; - } else { - m_tile[i] = 1; - } + for (int i = rank_start; i != rank_end; i += increment) { + const index_type length = m_upper[i] - m_lower[i]; + if (m_tile[i] <= 0) { + m_tune_tile_size = true; + if ((inner_direction == Iterate::Right && (i < rank - 1)) || + (inner_direction == Iterate::Left && (i > 0))) { + if (m_prod_tile_dims * properties.default_tile_size < + static_cast(properties.max_total_tile_size)) { + m_tile[i] = properties.default_tile_size; } else { - m_tile[i] = 16; + m_tile[i] = 1; } - } - m_tile_end[i] = - static_cast((span + m_tile[i] - 1) / m_tile[i]); - m_num_tiles *= m_tile_end[i]; - m_prod_tile_dims *= m_tile[i]; - } - if (m_prod_tile_dims > - 1024) { // Match Cuda restriction for ParallelReduce; 1024,1024,64 - // max per dim (Kepler), but product num_threads < 1024 - if (is_cuda_exec_space) { - printf(" Tile dimensions exceed Cuda limits\n"); - Kokkos::abort( - "Cuda ExecSpace Error: MDRange tile dims exceed maximum number " - "of threads per block - choose smaller tile dims"); } else { - printf(" Tile dimensions exceed HIP limits\n"); - Kokkos::abort( - "HIP ExecSpace Error: MDRange tile dims exceed maximum number of " - "threads per block - choose smaller tile dims"); + m_tile[i] = properties.default_largest_tile_size == 0 + ? std::max(length, 1) + : properties.default_largest_tile_size; } } + m_tile_end[i] = + static_cast((length + m_tile[i] - 1) / m_tile[i]); + m_num_tiles *= m_tile_end[i]; + m_prod_tile_dims *= m_tile[i]; + } + if (m_prod_tile_dims > static_cast(properties.max_threads)) { + printf(" Product of tile dimensions exceed maximum limit: %d\n", + static_cast(properties.max_threads)); + Kokkos::abort( + "ExecSpace Error: MDRange tile dims exceed maximum number " + "of threads per block - choose smaller tile dims"); } -#endif } }; diff --git a/lib/kokkos/core/src/Kokkos_AnonymousSpace.hpp b/lib/kokkos/core/src/Kokkos_AnonymousSpace.hpp index 8e226a078d..fb94049d7a 100644 --- a/lib/kokkos/core/src/Kokkos_AnonymousSpace.hpp +++ b/lib/kokkos/core/src/Kokkos_AnonymousSpace.hpp @@ -104,20 +104,6 @@ struct MemorySpaceAccess { enum : bool { deepcopy = true }; }; -template -struct VerifyExecutionCanAccessMemorySpace { - enum { value = 1 }; - KOKKOS_INLINE_FUNCTION static void verify(void) {} - KOKKOS_INLINE_FUNCTION static void verify(const void *) {} -}; - -template -struct VerifyExecutionCanAccessMemorySpace { - enum { value = 1 }; - KOKKOS_INLINE_FUNCTION static void verify(void) {} - KOKKOS_INLINE_FUNCTION static void verify(const void *) {} -}; - } // namespace Impl } // namespace Kokkos diff --git a/lib/kokkos/core/src/Kokkos_Complex.hpp b/lib/kokkos/core/src/Kokkos_Complex.hpp index fb2925a066..6578723fc8 100644 --- a/lib/kokkos/core/src/Kokkos_Complex.hpp +++ b/lib/kokkos/core/src/Kokkos_Complex.hpp @@ -45,14 +45,13 @@ #define KOKKOS_COMPLEX_HPP #include +#include #include +#include #include +#include #include -#ifdef KOKKOS_ENABLE_SYCL -#include -#endif - namespace Kokkos { /// \class complex @@ -220,10 +219,11 @@ class // Conditional noexcept, just in case RType throws on divide-by-zero KOKKOS_CONSTEXPR_14 KOKKOS_INLINE_FUNCTION complex& operator/=( const complex& y) noexcept(noexcept(RealType{} / RealType{})) { + using Kokkos::Experimental::fabs; // Scale (by the "1-norm" of y) to avoid unwarranted overflow. // If the real part is +/-Inf and the imaginary part is -/+Inf, // this won't change the result. - const RealType s = std::fabs(y.real()) + std::fabs(y.imag()); + const RealType s = fabs(y.real()) + fabs(y.imag()); // If s is 0, then y is zero, so x/y == real(x)/0 + i*imag(x)/0. // In that case, the relation x/y == (x/s) / (y/s) doesn't hold, @@ -248,10 +248,11 @@ class KOKKOS_INLINE_FUNCTION complex& operator/=( const std::complex& y) noexcept(noexcept(RealType{} / RealType{})) { + using Kokkos::Experimental::fabs; // Scale (by the "1-norm" of y) to avoid unwarranted overflow. // If the real part is +/-Inf and the imaginary part is -/+Inf, // this won't change the result. - const RealType s = std::fabs(y.real()) + std::fabs(y.imag()); + const RealType s = fabs(y.real()) + fabs(y.imag()); // If s is 0, then y is zero, so x/y == real(x)/0 + i*imag(x)/0. // In that case, the relation x/y == (x/s) / (y/s) doesn't hold, @@ -693,35 +694,96 @@ KOKKOS_INLINE_FUNCTION RealType real(const complex& x) noexcept { return x.real(); } +//! Constructs a complex number from magnitude and phase angle +template +KOKKOS_INLINE_FUNCTION complex polar(const T& r, const T& theta = T()) { + using Kokkos::Experimental::cos; + using Kokkos::Experimental::sin; + KOKKOS_EXPECTS(r >= 0); + return complex(r * cos(theta), r * sin(theta)); +} + //! Absolute value (magnitude) of a complex number. template KOKKOS_INLINE_FUNCTION RealType abs(const complex& x) { -#ifdef KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_SYCL - using cl::sycl::hypot; -#else - using std::hypot; -#endif + using Kokkos::Experimental::hypot; return hypot(x.real(), x.imag()); } //! Power of a complex number -template -KOKKOS_INLINE_FUNCTION Kokkos::complex pow(const complex& x, - const RealType& e) { - RealType r = abs(x); -#ifdef KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_SYCL - using cl::sycl::atan; - using cl::sycl::cos; - using cl::sycl::pow; - using cl::sycl::sin; -#else - using std::atan; - using std::cos; - using std::pow; - using std::sin; -#endif - RealType phi = atan(x.imag() / x.real()); - return pow(r, e) * Kokkos::complex(cos(phi * e), sin(phi * e)); +template +KOKKOS_INLINE_FUNCTION complex pow(const complex& x, const T& y) { + using Kokkos::Experimental::atan2; + using Kokkos::Experimental::pow; + T r = abs(x); + T theta = atan2(x.imag(), x.real()); + return polar(pow(r, y), y * theta); +} + +template +KOKKOS_INLINE_FUNCTION complex pow(const T& x, const complex& y) { + return pow(complex(x), y); +} + +template +KOKKOS_INLINE_FUNCTION complex pow(const complex& x, + const complex& y) { + using Kokkos::Experimental::log; + + return x == T() ? T() : exp(y * log(x)); +} + +namespace Impl { +// NOTE promote would also be useful for math functions +template ::value> +struct promote { + using type = double; +}; +template +struct promote {}; +template <> +struct promote { + using type = long double; +}; +template <> +struct promote { + using type = double; +}; +template <> +struct promote { + using type = float; +}; +template +using promote_t = typename promote::type; +template +struct promote_2 { + using type = decltype(promote_t() + promote_t()); +}; +template +using promote_2_t = typename promote_2::type; +} // namespace Impl + +template ::value>> +KOKKOS_INLINE_FUNCTION complex> pow( + const T& x, const complex& y) { + using type = Impl::promote_2_t; + return pow(type(x), complex(y)); +} + +template ::value>> +KOKKOS_INLINE_FUNCTION complex> pow(const complex& x, + const U& y) { + using type = Impl::promote_2_t; + return pow(complex(x), type(y)); +} + +template +KOKKOS_INLINE_FUNCTION complex> pow( + const complex& x, const complex& y) { + using type = Impl::promote_2_t; + return pow(complex(x), complex(y)); } //! Square root of a complex number. This is intended to match the stdc++ @@ -729,26 +791,21 @@ KOKKOS_INLINE_FUNCTION Kokkos::complex pow(const complex& x, template KOKKOS_INLINE_FUNCTION Kokkos::complex sqrt( const complex& x) { -#ifdef KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_SYCL - using cl::sycl::abs; - using cl::sycl::sqrt; -#else - using std::abs; - using std::sqrt; -#endif + using Kokkos::Experimental::fabs; + using Kokkos::Experimental::sqrt; RealType r = x.real(); RealType i = x.imag(); if (r == RealType()) { - RealType t = sqrt(abs(i) / 2); + RealType t = sqrt(fabs(i) / 2); return Kokkos::complex(t, i < RealType() ? -t : t); } else { - RealType t = sqrt(2 * (abs(x) + abs(r))); + RealType t = sqrt(2 * (abs(x) + fabs(r))); RealType u = t / 2; - return r > RealType() - ? Kokkos::complex(u, i / t) - : Kokkos::complex(abs(i) / t, i < RealType() ? -u : u); + return r > RealType() ? Kokkos::complex(u, i / t) + : Kokkos::complex(fabs(i) / t, + i < RealType() ? -u : u); } } @@ -762,15 +819,9 @@ KOKKOS_INLINE_FUNCTION complex conj( //! Exponential of a complex number. template KOKKOS_INLINE_FUNCTION complex exp(const complex& x) { -#ifdef KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_SYCL - using cl::sycl::cos; - using cl::sycl::exp; - using cl::sycl::sin; -#else - using std::cos; - using std::exp; - using std::sin; -#endif + using Kokkos::Experimental::cos; + using Kokkos::Experimental::exp; + using Kokkos::Experimental::sin; return exp(x.real()) * complex(cos(x.imag()), sin(x.imag())); } @@ -778,14 +829,9 @@ KOKKOS_INLINE_FUNCTION complex exp(const complex& x) { template KOKKOS_INLINE_FUNCTION Kokkos::complex log( const complex& x) { -#ifdef KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_SYCL - using cl::sycl::atan; - using cl::sycl::log; -#else - using std::atan; - using std::log; -#endif - RealType phi = atan(x.imag() / x.real()); + using Kokkos::Experimental::atan2; + using Kokkos::Experimental::log; + RealType phi = atan2(x.imag(), x.real()); return Kokkos::complex(log(abs(x)), phi); } @@ -793,17 +839,10 @@ KOKKOS_INLINE_FUNCTION Kokkos::complex log( template KOKKOS_INLINE_FUNCTION Kokkos::complex sin( const complex& x) { -#ifdef KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_SYCL - using cl::sycl::cos; - using cl::sycl::cosh; - using cl::sycl::sin; - using cl::sycl::sinh; -#else - using std::cos; - using std::cosh; - using std::sin; - using std::sinh; -#endif + using Kokkos::Experimental::cos; + using Kokkos::Experimental::cosh; + using Kokkos::Experimental::sin; + using Kokkos::Experimental::sinh; return Kokkos::complex(sin(x.real()) * cosh(x.imag()), cos(x.real()) * sinh(x.imag())); } @@ -812,17 +851,10 @@ KOKKOS_INLINE_FUNCTION Kokkos::complex sin( template KOKKOS_INLINE_FUNCTION Kokkos::complex cos( const complex& x) { -#ifdef KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_SYCL - using cl::sycl::cos; - using cl::sycl::cosh; - using cl::sycl::sin; - using cl::sycl::sinh; -#else - using std::cos; - using std::cosh; - using std::sin; - using std::sinh; -#endif + using Kokkos::Experimental::cos; + using Kokkos::Experimental::cosh; + using Kokkos::Experimental::sin; + using Kokkos::Experimental::sinh; return Kokkos::complex(cos(x.real()) * cosh(x.imag()), -sin(x.real()) * sinh(x.imag())); } @@ -838,17 +870,10 @@ KOKKOS_INLINE_FUNCTION Kokkos::complex tan( template KOKKOS_INLINE_FUNCTION Kokkos::complex sinh( const complex& x) { -#ifdef KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_SYCL - using cl::sycl::cos; - using cl::sycl::cosh; - using cl::sycl::sin; - using cl::sycl::sinh; -#else - using std::cos; - using std::cosh; - using std::sin; - using std::sinh; -#endif + using Kokkos::Experimental::cos; + using Kokkos::Experimental::cosh; + using Kokkos::Experimental::sin; + using Kokkos::Experimental::sinh; return Kokkos::complex(sinh(x.real()) * cos(x.imag()), cosh(x.real()) * sin(x.imag())); } @@ -857,17 +882,10 @@ KOKKOS_INLINE_FUNCTION Kokkos::complex sinh( template KOKKOS_INLINE_FUNCTION Kokkos::complex cosh( const complex& x) { -#ifdef KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_SYCL - using cl::sycl::cos; - using cl::sycl::cosh; - using cl::sycl::sin; - using cl::sycl::sinh; -#else - using std::cos; - using std::cosh; - using std::sin; - using std::sinh; -#endif + using Kokkos::Experimental::cos; + using Kokkos::Experimental::cosh; + using Kokkos::Experimental::sin; + using Kokkos::Experimental::sinh; return Kokkos::complex(cosh(x.real()) * cos(x.imag()), sinh(x.real()) * sin(x.imag())); } @@ -898,13 +916,8 @@ KOKKOS_INLINE_FUNCTION Kokkos::complex acosh( template KOKKOS_INLINE_FUNCTION Kokkos::complex atanh( const complex& x) { -#ifdef KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_SYCL - using cl::sycl::atan2; - using cl::sycl::log; -#else - using std::atan2; - using std::log; -#endif + using Kokkos::Experimental::atan2; + using Kokkos::Experimental::log; const RealType i2 = x.imag() * x.imag(); const RealType r = RealType(1.0) - i2 - x.real() * x.real(); @@ -933,12 +946,7 @@ KOKKOS_INLINE_FUNCTION Kokkos::complex asin( template KOKKOS_INLINE_FUNCTION Kokkos::complex acos( const complex& x) { -#ifdef KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_SYCL - using cl::sycl::acos; - -#else - using std::acos; -#endif + using Kokkos::Experimental::acos; Kokkos::complex t = asin(x); RealType pi_2 = acos(RealType(0.0)); return Kokkos::complex(pi_2 - t.real(), -t.imag()); @@ -948,13 +956,8 @@ KOKKOS_INLINE_FUNCTION Kokkos::complex acos( template KOKKOS_INLINE_FUNCTION Kokkos::complex atan( const complex& x) { -#ifdef KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_SYCL - using cl::sycl::atan2; - using cl::sycl::log; -#else - using std::atan2; - using std::log; -#endif + using Kokkos::Experimental::atan2; + using Kokkos::Experimental::log; const RealType r2 = x.real() * x.real(); const RealType i = RealType(1.0) - r2 - x.imag() * x.imag(); @@ -996,12 +999,13 @@ KOKKOS_INLINE_FUNCTION operator/(const complex& x, const complex& y) noexcept(noexcept(RealType1{} / RealType2{})) { + using Kokkos::Experimental::fabs; // Scale (by the "1-norm" of y) to avoid unwarranted overflow. // If the real part is +/-Inf and the imaginary part is -/+Inf, // this won't change the result. using common_real_type = typename std::common_type::type; - const common_real_type s = std::fabs(real(y)) + std::fabs(imag(y)); + const common_real_type s = fabs(real(y)) + fabs(imag(y)); // If s is 0, then y is zero, so x/y == real(x)/0 + i*imag(x)/0. // In that case, the relation x/y == (x/s) / (y/s) doesn't hold, @@ -1046,7 +1050,7 @@ std::istream& operator>>(std::istream& is, complex& x) { } template -struct reduction_identity > { +struct reduction_identity> { using t_red_ident = reduction_identity; KOKKOS_FORCEINLINE_FUNCTION constexpr static Kokkos::complex sum() noexcept { diff --git a/lib/kokkos/core/src/Kokkos_Core.hpp b/lib/kokkos/core/src/Kokkos_Core.hpp index 4dac463a66..c3771ab393 100644 --- a/lib/kokkos/core/src/Kokkos_Core.hpp +++ b/lib/kokkos/core/src/Kokkos_Core.hpp @@ -58,6 +58,7 @@ #include #include #include +#include #include #include #include @@ -86,6 +87,10 @@ struct InitArguments { int skip_device; bool disable_warnings; bool tune_internals; + bool tool_help = false; + std::string tool_lib = {}; + std::string tool_args = {}; + InitArguments(int nt = -1, int nn = -1, int dv = -1, bool dw = false, bool ti = false) : num_threads{nt}, @@ -139,6 +144,10 @@ void pre_initialize(const InitArguments& args); void post_initialize(const InitArguments& args); +void declare_configuration_metadata(const std::string& category, + const std::string& key, + const std::string& value); + } // namespace Impl bool is_initialized() noexcept; diff --git a/lib/kokkos/core/src/Kokkos_Core_fwd.hpp b/lib/kokkos/core/src/Kokkos_Core_fwd.hpp index 7502719c73..fe7eba3f6e 100644 --- a/lib/kokkos/core/src/Kokkos_Core_fwd.hpp +++ b/lib/kokkos/core/src/Kokkos_Core_fwd.hpp @@ -50,6 +50,7 @@ // and compiler environment then sets a collection of #define macros. #include +#include #include #include @@ -180,7 +181,6 @@ using DefaultHostExecutionSpace KOKKOS_IMPL_DEFAULT_HOST_EXEC_SPACE_ANNOTATION = // a given memory space. namespace Kokkos { - namespace Impl { #if defined(KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_CUDA) && \ @@ -196,16 +196,22 @@ using ActiveExecutionMemorySpace = Kokkos::HostSpace; using ActiveExecutionMemorySpace = void; #endif -template -struct VerifyExecutionCanAccessMemorySpace { - enum { value = 0 }; +template +struct MemorySpaceAccess; + +template ::accessible> +struct verify_space { + KOKKOS_FUNCTION static void check() {} }; -template -struct VerifyExecutionCanAccessMemorySpace { - enum { value = 1 }; - KOKKOS_INLINE_FUNCTION static void verify(void) {} - KOKKOS_INLINE_FUNCTION static void verify(const void *) {} +template +struct verify_space { + KOKKOS_FUNCTION static void check() { + Kokkos::abort( + "Kokkos::View ERROR: attempt to access inaccessible memory space"); + }; }; // Base class for exec space initializer factories @@ -220,13 +226,13 @@ class LogicalMemorySpace; } // namespace Kokkos -#define KOKKOS_RESTRICT_EXECUTION_TO_DATA(DATA_SPACE, DATA_PTR) \ - Kokkos::Impl::VerifyExecutionCanAccessMemorySpace< \ - Kokkos::Impl::ActiveExecutionMemorySpace, DATA_SPACE>::verify(DATA_PTR) +#define KOKKOS_RESTRICT_EXECUTION_TO_DATA(DATA_SPACE, DATA_PTR) \ + Kokkos::Impl::verify_space::check(); -#define KOKKOS_RESTRICT_EXECUTION_TO_(DATA_SPACE) \ - Kokkos::Impl::VerifyExecutionCanAccessMemorySpace< \ - Kokkos::Impl::ActiveExecutionMemorySpace, DATA_SPACE>::verify() +#define KOKKOS_RESTRICT_EXECUTION_TO_(DATA_SPACE) \ + Kokkos::Impl::verify_space::check(); //---------------------------------------------------------------------------- @@ -256,8 +262,7 @@ template struct ViewCopy; -template +template struct FunctorPolicyExecutionSpace; //---------------------------------------------------------------------------- diff --git a/lib/kokkos/core/src/Kokkos_Crs.hpp b/lib/kokkos/core/src/Kokkos_Crs.hpp index 4a573d82c0..1a10500b19 100644 --- a/lib/kokkos/core/src/Kokkos_Crs.hpp +++ b/lib/kokkos/core/src/Kokkos_Crs.hpp @@ -199,7 +199,7 @@ class CrsRowMapFromCounts { public: KOKKOS_INLINE_FUNCTION void operator()(index_type i, value_type& update, bool final_pass) const { - if (i < m_in.size()) { + if (i < static_cast(m_in.size())) { update += m_in(i); if (final_pass) m_out(i + 1) = update; } else if (final_pass) { diff --git a/lib/kokkos/core/src/Kokkos_Cuda.hpp b/lib/kokkos/core/src/Kokkos_Cuda.hpp index 81e11f3f12..7a218120bb 100644 --- a/lib/kokkos/core/src/Kokkos_Cuda.hpp +++ b/lib/kokkos/core/src/Kokkos_Cuda.hpp @@ -63,6 +63,7 @@ #include #include #include +#include /*--------------------------------------------------------------------------*/ @@ -198,16 +199,6 @@ class Cuda { Cuda(); - KOKKOS_FUNCTION Cuda(Cuda&& other) noexcept; - - KOKKOS_FUNCTION Cuda(const Cuda& other); - - KOKKOS_FUNCTION Cuda& operator=(Cuda&& other) noexcept; - - KOKKOS_FUNCTION Cuda& operator=(const Cuda& other); - - KOKKOS_FUNCTION ~Cuda() noexcept; - Cuda(cudaStream_t stream); //-------------------------------------------------------------------------- @@ -253,13 +244,12 @@ class Cuda { static const char* name(); inline Impl::CudaInternal* impl_internal_space_instance() const { - return m_space_instance; + return m_space_instance.get(); } uint32_t impl_instance_id() const noexcept { return 0; } private: - Impl::CudaInternal* m_space_instance; - int* m_counter; + Kokkos::Impl::HostSharedPtr m_space_instance; }; namespace Tools { @@ -319,38 +309,8 @@ struct MemorySpaceAccess -struct VerifyExecutionCanAccessMemorySpace { - enum : bool { value = true }; - KOKKOS_INLINE_FUNCTION static void verify(void) {} - KOKKOS_INLINE_FUNCTION static void verify(const void*) {} -}; - -template <> -struct VerifyExecutionCanAccessMemorySpace { - enum : bool { value = false }; - inline static void verify(void) { CudaSpace::access_error(); } - inline static void verify(const void* p) { CudaSpace::access_error(p); } -}; - } // namespace Impl } // namespace Kokkos -/*--------------------------------------------------------------------------*/ -/*--------------------------------------------------------------------------*/ - -#include -#include -#include -#include -#include -#include -#include - -#include -//---------------------------------------------------------------------------- - #endif /* #if defined( KOKKOS_ENABLE_CUDA ) */ #endif /* #ifndef KOKKOS_CUDA_HPP */ diff --git a/lib/kokkos/core/src/Kokkos_CudaSpace.hpp b/lib/kokkos/core/src/Kokkos_CudaSpace.hpp index fc1c0e2f8a..e10fae93c7 100644 --- a/lib/kokkos/core/src/Kokkos_CudaSpace.hpp +++ b/lib/kokkos/core/src/Kokkos_CudaSpace.hpp @@ -53,8 +53,10 @@ #include #include #include +#include #include +#include #include @@ -119,8 +121,8 @@ class CudaSpace { /*--------------------------------*/ /** \brief Error reporting for HostSpace attempt to access CudaSpace */ - static void access_error(); - static void access_error(const void* const); + KOKKOS_DEPRECATED static void access_error(); + KOKKOS_DEPRECATED static void access_error(const void* const); private: int m_device; ///< Which Cuda device @@ -128,42 +130,6 @@ class CudaSpace { static constexpr const char* m_name = "Cuda"; friend class Kokkos::Impl::SharedAllocationRecord; }; - -namespace Impl { -/// \brief Initialize lock array for arbitrary size atomics. -/// -/// Arbitrary atomics are implemented using a hash table of locks -/// where the hash value is derived from the address of the -/// object for which an atomic operation is performed. -/// This function initializes the locks to zero (unset). -void init_lock_arrays_cuda_space(); - -/// \brief Retrieve the pointer to the lock array for arbitrary size atomics. -/// -/// Arbitrary atomics are implemented using a hash table of locks -/// where the hash value is derived from the address of the -/// object for which an atomic operation is performed. -/// This function retrieves the lock array pointer. -/// If the array is not yet allocated it will do so. -int* atomic_lock_array_cuda_space_ptr(bool deallocate = false); - -/// \brief Retrieve the pointer to the scratch array for team and thread private -/// global memory. -/// -/// Team and Thread private scratch allocations in -/// global memory are acquired via locks. -/// This function retrieves the lock array pointer. -/// If the array is not yet allocated it will do so. -int* scratch_lock_array_cuda_space_ptr(bool deallocate = false); - -/// \brief Retrieve the pointer to the scratch array for unique identifiers. -/// -/// Unique identifiers in the range 0-Cuda::concurrency -/// are provided via locks. -/// This function retrieves the lock array pointer. -/// If the array is not yet allocated it will do so. -int* threadid_lock_array_cuda_space_ptr(bool deallocate = false); -} // namespace Impl } // namespace Kokkos /*--------------------------------------------------------------------------*/ @@ -313,6 +279,11 @@ class CudaHostPinnedSpace { namespace Kokkos { namespace Impl { +cudaStream_t cuda_get_deep_copy_stream(); + +const std::unique_ptr& cuda_get_deep_copy_space( + bool initialize = true); + static_assert(Kokkos::Impl::MemorySpaceAccess::assignable, ""); @@ -784,104 +755,21 @@ struct DeepCopy { namespace Kokkos { namespace Impl { -/** Running in CudaSpace attempting to access HostSpace: error */ -template <> -struct VerifyExecutionCanAccessMemorySpace { - enum : bool { value = false }; - KOKKOS_INLINE_FUNCTION static void verify(void) { - Kokkos::abort("Cuda code attempted to access HostSpace memory"); - } - - KOKKOS_INLINE_FUNCTION static void verify(const void*) { - Kokkos::abort("Cuda code attempted to access HostSpace memory"); - } -}; - -/** Running in CudaSpace accessing CudaUVMSpace: ok */ -template <> -struct VerifyExecutionCanAccessMemorySpace { - enum : bool { value = true }; - KOKKOS_INLINE_FUNCTION static void verify(void) {} - KOKKOS_INLINE_FUNCTION static void verify(const void*) {} -}; - -/** Running in CudaSpace accessing CudaHostPinnedSpace: ok */ -template <> -struct VerifyExecutionCanAccessMemorySpace { - enum : bool { value = true }; - KOKKOS_INLINE_FUNCTION static void verify(void) {} - KOKKOS_INLINE_FUNCTION static void verify(const void*) {} -}; - -/** Running in CudaSpace attempting to access an unknown space: error */ -template -struct VerifyExecutionCanAccessMemorySpace< - typename std::enable_if::value, - Kokkos::CudaSpace>::type, - OtherSpace> { - enum : bool { value = false }; - KOKKOS_INLINE_FUNCTION static void verify(void) { - Kokkos::abort("Cuda code attempted to access unknown Space memory"); - } - - KOKKOS_INLINE_FUNCTION static void verify(const void*) { - Kokkos::abort("Cuda code attempted to access unknown Space memory"); - } -}; - -//---------------------------------------------------------------------------- -/** Running in HostSpace attempting to access CudaSpace */ -template <> -struct VerifyExecutionCanAccessMemorySpace { - enum : bool { value = false }; - inline static void verify(void) { CudaSpace::access_error(); } - inline static void verify(const void* p) { CudaSpace::access_error(p); } -}; - -/** Running in HostSpace accessing CudaUVMSpace is OK */ -template <> -struct VerifyExecutionCanAccessMemorySpace { - enum : bool { value = true }; - inline static void verify(void) {} - inline static void verify(const void*) {} -}; - -/** Running in HostSpace accessing CudaHostPinnedSpace is OK */ -template <> -struct VerifyExecutionCanAccessMemorySpace { - enum : bool { value = true }; - KOKKOS_INLINE_FUNCTION static void verify(void) {} - KOKKOS_INLINE_FUNCTION static void verify(const void*) {} -}; - -} // namespace Impl -} // namespace Kokkos - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -namespace Kokkos { -namespace Impl { - template <> class SharedAllocationRecord - : public SharedAllocationRecord { + : public HostInaccessibleSharedAllocationRecordCommon { private: friend class SharedAllocationRecord; + friend class SharedAllocationRecordCommon; + friend class HostInaccessibleSharedAllocationRecordCommon; using RecordBase = SharedAllocationRecord; + using base_t = + HostInaccessibleSharedAllocationRecordCommon; SharedAllocationRecord(const SharedAllocationRecord&) = delete; SharedAllocationRecord& operator=(const SharedAllocationRecord&) = delete; - static void deallocate(RecordBase*); - static ::cudaTextureObject_t attach_texture_object( const unsigned sizeof_alias, void* const alloc_ptr, const size_t alloc_size); @@ -890,39 +778,19 @@ class SharedAllocationRecord static RecordBase s_root_record; #endif - ::cudaTextureObject_t m_tex_obj; + ::cudaTextureObject_t m_tex_obj = 0; const Kokkos::CudaSpace m_space; protected: ~SharedAllocationRecord(); - SharedAllocationRecord() : RecordBase(), m_tex_obj(0), m_space() {} + SharedAllocationRecord() = default; SharedAllocationRecord( const Kokkos::CudaSpace& arg_space, const std::string& arg_label, const size_t arg_alloc_size, - const RecordBase::function_type arg_dealloc = &deallocate); + const RecordBase::function_type arg_dealloc = &base_t::deallocate); public: - std::string get_label() const; - - static SharedAllocationRecord* allocate(const Kokkos::CudaSpace& arg_space, - const std::string& arg_label, - const size_t arg_alloc_size); - - /**\brief Allocate tracked memory in the space */ - static void* allocate_tracked(const Kokkos::CudaSpace& arg_space, - const std::string& arg_label, - const size_t arg_alloc_size); - - /**\brief Reallocate tracked memory in the space */ - static void* reallocate_tracked(void* const arg_alloc_ptr, - const size_t arg_alloc_size); - - /**\brief Deallocate tracked memory in the space */ - static void deallocate_tracked(void* const arg_alloc_ptr); - - static SharedAllocationRecord* get_record(void* arg_alloc_ptr); - template inline ::cudaTextureObject_t attach_texture_object() { static_assert((std::is_same::value || @@ -945,57 +813,35 @@ class SharedAllocationRecord // Texture object is attached to the entire allocation range return ptr - reinterpret_cast(RecordBase::m_alloc_ptr); } - - static void print_records(std::ostream&, const Kokkos::CudaSpace&, - bool detail = false); }; template <> class SharedAllocationRecord - : public SharedAllocationRecord { + : public SharedAllocationRecordCommon { private: + friend class SharedAllocationRecordCommon; + + using base_t = SharedAllocationRecordCommon; using RecordBase = SharedAllocationRecord; SharedAllocationRecord(const SharedAllocationRecord&) = delete; SharedAllocationRecord& operator=(const SharedAllocationRecord&) = delete; - static void deallocate(RecordBase*); - static RecordBase s_root_record; - ::cudaTextureObject_t m_tex_obj; + ::cudaTextureObject_t m_tex_obj = 0; const Kokkos::CudaUVMSpace m_space; protected: ~SharedAllocationRecord(); - SharedAllocationRecord() : RecordBase(), m_tex_obj(0), m_space() {} + SharedAllocationRecord() = default; SharedAllocationRecord( const Kokkos::CudaUVMSpace& arg_space, const std::string& arg_label, const size_t arg_alloc_size, - const RecordBase::function_type arg_dealloc = &deallocate); + const RecordBase::function_type arg_dealloc = &base_t::deallocate); public: - std::string get_label() const; - - static SharedAllocationRecord* allocate(const Kokkos::CudaUVMSpace& arg_space, - const std::string& arg_label, - const size_t arg_alloc_size); - - /**\brief Allocate tracked memory in the space */ - static void* allocate_tracked(const Kokkos::CudaUVMSpace& arg_space, - const std::string& arg_label, - const size_t arg_alloc_size); - - /**\brief Reallocate tracked memory in the space */ - static void* reallocate_tracked(void* const arg_alloc_ptr, - const size_t arg_alloc_size); - - /**\brief Deallocate tracked memory in the space */ - static void deallocate_tracked(void* const arg_alloc_ptr); - - static SharedAllocationRecord* get_record(void* arg_alloc_ptr); - template inline ::cudaTextureObject_t attach_texture_object() { static_assert((std::is_same::value || @@ -1019,57 +865,32 @@ class SharedAllocationRecord // Texture object is attached to the entire allocation range return ptr - reinterpret_cast(RecordBase::m_alloc_ptr); } - - static void print_records(std::ostream&, const Kokkos::CudaUVMSpace&, - bool detail = false); }; template <> class SharedAllocationRecord - : public SharedAllocationRecord { + : public SharedAllocationRecordCommon { private: + friend class SharedAllocationRecordCommon; + using RecordBase = SharedAllocationRecord; + using base_t = SharedAllocationRecordCommon; SharedAllocationRecord(const SharedAllocationRecord&) = delete; SharedAllocationRecord& operator=(const SharedAllocationRecord&) = delete; - static void deallocate(RecordBase*); - static RecordBase s_root_record; const Kokkos::CudaHostPinnedSpace m_space; protected: ~SharedAllocationRecord(); - SharedAllocationRecord() : RecordBase(), m_space() {} + SharedAllocationRecord() = default; SharedAllocationRecord( const Kokkos::CudaHostPinnedSpace& arg_space, const std::string& arg_label, const size_t arg_alloc_size, const RecordBase::function_type arg_dealloc = &deallocate); - - public: - std::string get_label() const; - - static SharedAllocationRecord* allocate( - const Kokkos::CudaHostPinnedSpace& arg_space, - const std::string& arg_label, const size_t arg_alloc_size); - /**\brief Allocate tracked memory in the space */ - static void* allocate_tracked(const Kokkos::CudaHostPinnedSpace& arg_space, - const std::string& arg_label, - const size_t arg_alloc_size); - - /**\brief Reallocate tracked memory in the space */ - static void* reallocate_tracked(void* const arg_alloc_ptr, - const size_t arg_alloc_size); - - /**\brief Deallocate tracked memory in the space */ - static void deallocate_tracked(void* const arg_alloc_ptr); - - static SharedAllocationRecord* get_record(void* arg_alloc_ptr); - - static void print_records(std::ostream&, const Kokkos::CudaHostPinnedSpace&, - bool detail = false); }; } // namespace Impl diff --git a/lib/kokkos/core/src/Kokkos_ExecPolicy.hpp b/lib/kokkos/core/src/Kokkos_ExecPolicy.hpp index 3afe081701..55aed13670 100644 --- a/lib/kokkos/core/src/Kokkos_ExecPolicy.hpp +++ b/lib/kokkos/core/src/Kokkos_ExecPolicy.hpp @@ -856,11 +856,12 @@ KOKKOS_INLINE_FUNCTION_DELETED Impl::ThreadVectorRangeBoundariesStruct ThreadVectorRange(const TeamMemberType&, const iType& count) = delete; -template -KOKKOS_INLINE_FUNCTION_DELETED - Impl::ThreadVectorRangeBoundariesStruct - ThreadVectorRange(const TeamMemberType&, const iType& arg_begin, - const iType& arg_end) = delete; +template +KOKKOS_INLINE_FUNCTION_DELETED Impl::ThreadVectorRangeBoundariesStruct< + typename std::common_type::type, TeamMemberType> +ThreadVectorRange(const TeamMemberType&, const iType1& arg_begin, + const iType2& arg_end) = delete; namespace Impl { @@ -902,85 +903,6 @@ struct ParallelConstructName { } // namespace Kokkos namespace Kokkos { -namespace Experimental { - -namespace Impl { -template -struct PolicyPropertyAdaptor; - -template class Policy, - class... Properties> -struct PolicyPropertyAdaptor, - Policy> { - using policy_in_t = Policy; - static_assert(is_execution_policy::value, ""); - using policy_out_t = Policy, - typename policy_in_t::traits::occupancy_control>; -}; - -template