mirror of https://github.com/vllm-project/vllm
319 lines
10 KiB
CMake
319 lines
10 KiB
CMake
cmake_minimum_required(VERSION 3.26)
|
|
|
|
project(vllm_extensions LANGUAGES CXX)
|
|
|
|
# CUDA by default, can be overridden by using -DVLLM_TARGET_DEVICE=... (used by setup.py)
|
|
set(VLLM_TARGET_DEVICE "cuda" CACHE STRING "Target device backend for vLLM")
|
|
|
|
message(STATUS "Build type: ${CMAKE_BUILD_TYPE}")
|
|
message(STATUS "Target device: ${VLLM_TARGET_DEVICE}")
|
|
|
|
include(${CMAKE_CURRENT_LIST_DIR}/cmake/utils.cmake)
|
|
|
|
# Suppress potential warnings about unused manually-specified variables
|
|
set(ignoreMe "${VLLM_PYTHON_PATH}")
|
|
|
|
#
|
|
# Supported python versions. These versions will be searched in order, the
|
|
# first match will be selected. These should be kept in sync with setup.py.
|
|
#
|
|
set(PYTHON_SUPPORTED_VERSIONS "3.8" "3.9" "3.10" "3.11" "3.12")
|
|
|
|
# Supported NVIDIA architectures.
|
|
set(CUDA_SUPPORTED_ARCHS "7.0;7.5;8.0;8.6;8.9;9.0")
|
|
|
|
# Supported AMD GPU architectures.
|
|
set(HIP_SUPPORTED_ARCHS "gfx906;gfx908;gfx90a;gfx940;gfx941;gfx942;gfx1030;gfx1100")
|
|
|
|
#
|
|
# Supported/expected torch versions for CUDA/ROCm.
|
|
#
|
|
# Currently, having an incorrect pytorch version results in a warning
|
|
# rather than an error.
|
|
#
|
|
# Note: the CUDA torch version is derived from pyproject.toml and various
|
|
# requirements.txt files and should be kept consistent. The ROCm torch
|
|
# versions are derived from Dockerfile.rocm
|
|
#
|
|
set(TORCH_SUPPORTED_VERSION_CUDA "2.4.0")
|
|
set(TORCH_SUPPORTED_VERSION_ROCM "2.5.0")
|
|
|
|
#
|
|
# Try to find python package with an executable that exactly matches
|
|
# `VLLM_PYTHON_EXECUTABLE` and is one of the supported versions.
|
|
#
|
|
if (VLLM_PYTHON_EXECUTABLE)
|
|
find_python_from_executable(${VLLM_PYTHON_EXECUTABLE} "${PYTHON_SUPPORTED_VERSIONS}")
|
|
else()
|
|
message(FATAL_ERROR
|
|
"Please set VLLM_PYTHON_EXECUTABLE to the path of the desired python version"
|
|
" before running cmake configure.")
|
|
endif()
|
|
|
|
#
|
|
# Update cmake's `CMAKE_PREFIX_PATH` with torch location.
|
|
#
|
|
append_cmake_prefix_path("torch" "torch.utils.cmake_prefix_path")
|
|
|
|
# Ensure the 'nvcc' command is in the PATH
|
|
find_program(NVCC_EXECUTABLE nvcc)
|
|
if (CUDA_FOUND AND NOT NVCC_EXECUTABLE)
|
|
message(FATAL_ERROR "nvcc not found")
|
|
endif()
|
|
|
|
#
|
|
# Import torch cmake configuration.
|
|
# Torch also imports CUDA (and partially HIP) languages with some customizations,
|
|
# so there is no need to do this explicitly with check_language/enable_language,
|
|
# etc.
|
|
#
|
|
find_package(Torch REQUIRED)
|
|
|
|
#
|
|
# Add the `default` target which detects which extensions should be
|
|
# built based on platform/architecture. This is the same logic that
|
|
# setup.py uses to select which extensions should be built and should
|
|
# be kept in sync.
|
|
#
|
|
# The `default` target makes direct use of cmake easier since knowledge
|
|
# of which extensions are supported has been factored in, e.g.
|
|
#
|
|
# mkdir build && cd build
|
|
# cmake -G Ninja -DVLLM_PYTHON_EXECUTABLE=`which python3` -DCMAKE_LIBRARY_OUTPUT_DIRECTORY=../vllm ..
|
|
# cmake --build . --target default
|
|
#
|
|
add_custom_target(default)
|
|
message(STATUS "Enabling core extension.")
|
|
|
|
# Define _core_C extension
|
|
# built for (almost) every target platform, (excludes TPU and Neuron)
|
|
|
|
set(VLLM_EXT_SRC
|
|
"csrc/core/torch_bindings.cpp")
|
|
|
|
define_gpu_extension_target(
|
|
_core_C
|
|
DESTINATION vllm
|
|
LANGUAGE CXX
|
|
SOURCES ${VLLM_EXT_SRC}
|
|
COMPILE_FLAGS ${CXX_COMPILE_FLAGS}
|
|
USE_SABI 3
|
|
WITH_SOABI)
|
|
|
|
add_dependencies(default _core_C)
|
|
|
|
#
|
|
# Forward the non-CUDA device extensions to external CMake scripts.
|
|
#
|
|
if (NOT VLLM_TARGET_DEVICE STREQUAL "cuda" AND
|
|
NOT VLLM_TARGET_DEVICE STREQUAL "rocm")
|
|
if (VLLM_TARGET_DEVICE STREQUAL "cpu")
|
|
include(${CMAKE_CURRENT_LIST_DIR}/cmake/cpu_extension.cmake)
|
|
else()
|
|
return()
|
|
endif()
|
|
return()
|
|
endif()
|
|
|
|
#
|
|
# Set up GPU language and check the torch version and warn if it isn't
|
|
# what is expected.
|
|
#
|
|
if (NOT HIP_FOUND AND CUDA_FOUND)
|
|
set(VLLM_GPU_LANG "CUDA")
|
|
|
|
if (NOT Torch_VERSION VERSION_EQUAL ${TORCH_SUPPORTED_VERSION_CUDA})
|
|
message(WARNING "Pytorch version ${TORCH_SUPPORTED_VERSION_CUDA} "
|
|
"expected for CUDA build, saw ${Torch_VERSION} instead.")
|
|
endif()
|
|
elseif(HIP_FOUND)
|
|
set(VLLM_GPU_LANG "HIP")
|
|
|
|
# Importing torch recognizes and sets up some HIP/ROCm configuration but does
|
|
# not let cmake recognize .hip files. In order to get cmake to understand the
|
|
# .hip extension automatically, HIP must be enabled explicitly.
|
|
enable_language(HIP)
|
|
|
|
# ROCm 5.X and 6.X
|
|
if (ROCM_VERSION_DEV_MAJOR GREATER_EQUAL 5 AND
|
|
NOT Torch_VERSION VERSION_EQUAL ${TORCH_SUPPORTED_VERSION_ROCM})
|
|
message(WARNING "Pytorch version >= ${TORCH_SUPPORTED_VERSION_ROCM} "
|
|
"expected for ROCm build, saw ${Torch_VERSION} instead.")
|
|
endif()
|
|
else()
|
|
message(FATAL_ERROR "Can't find CUDA or HIP installation.")
|
|
endif()
|
|
|
|
#
|
|
# Override the GPU architectures detected by cmake/torch and filter them by
|
|
# the supported versions for the current language.
|
|
# The final set of arches is stored in `VLLM_GPU_ARCHES`.
|
|
#
|
|
override_gpu_arches(VLLM_GPU_ARCHES
|
|
${VLLM_GPU_LANG}
|
|
"${${VLLM_GPU_LANG}_SUPPORTED_ARCHS}")
|
|
|
|
#
|
|
# Query torch for additional GPU compilation flags for the given
|
|
# `VLLM_GPU_LANG`.
|
|
# The final set of arches is stored in `VLLM_GPU_FLAGS`.
|
|
#
|
|
get_torch_gpu_compiler_flags(VLLM_GPU_FLAGS ${VLLM_GPU_LANG})
|
|
|
|
#
|
|
# Set nvcc parallelism.
|
|
#
|
|
if(NVCC_THREADS AND VLLM_GPU_LANG STREQUAL "CUDA")
|
|
list(APPEND VLLM_GPU_FLAGS "--threads=${NVCC_THREADS}")
|
|
endif()
|
|
|
|
#
|
|
# Define other extension targets
|
|
#
|
|
|
|
#
|
|
# _C extension
|
|
#
|
|
|
|
set(VLLM_EXT_SRC
|
|
"csrc/cache_kernels.cu"
|
|
"csrc/attention/attention_kernels.cu"
|
|
"csrc/pos_encoding_kernels.cu"
|
|
"csrc/activation_kernels.cu"
|
|
"csrc/layernorm_kernels.cu"
|
|
"csrc/quantization/squeezellm/quant_cuda_kernel.cu"
|
|
"csrc/quantization/gptq/q_gemm.cu"
|
|
"csrc/quantization/compressed_tensors/int8_quant_kernels.cu"
|
|
"csrc/quantization/fp8/common.cu"
|
|
"csrc/cuda_utils_kernels.cu"
|
|
"csrc/moe_align_block_size_kernels.cu"
|
|
"csrc/prepare_inputs/advance_step.cu"
|
|
"csrc/torch_bindings.cpp")
|
|
|
|
if(VLLM_GPU_LANG STREQUAL "CUDA")
|
|
include(FetchContent)
|
|
SET(CUTLASS_ENABLE_HEADERS_ONLY ON CACHE BOOL "Enable only the header library")
|
|
FetchContent_Declare(
|
|
cutlass
|
|
GIT_REPOSITORY https://github.com/nvidia/cutlass.git
|
|
# CUTLASS 3.5.1
|
|
GIT_TAG 06b21349bcf6ddf6a1686a47a137ad1446579db9
|
|
GIT_PROGRESS TRUE
|
|
)
|
|
FetchContent_MakeAvailable(cutlass)
|
|
|
|
list(APPEND VLLM_EXT_SRC
|
|
"csrc/quantization/aqlm/gemm_kernels.cu"
|
|
"csrc/quantization/awq/gemm_kernels.cu"
|
|
"csrc/quantization/marlin/dense/marlin_cuda_kernel.cu"
|
|
"csrc/quantization/marlin/sparse/marlin_24_cuda_kernel.cu"
|
|
"csrc/quantization/marlin/qqq/marlin_qqq_gemm_kernel.cu"
|
|
"csrc/quantization/gptq_marlin/gptq_marlin.cu"
|
|
"csrc/quantization/gptq_marlin/gptq_marlin_repack.cu"
|
|
"csrc/quantization/gptq_marlin/awq_marlin_repack.cu"
|
|
"csrc/quantization/gguf/gguf_kernel.cu"
|
|
"csrc/quantization/fp8/fp8_marlin.cu"
|
|
"csrc/custom_all_reduce.cu"
|
|
"csrc/quantization/cutlass_w8a8/scaled_mm_entry.cu"
|
|
"csrc/quantization/cutlass_w8a8/scaled_mm_c2x.cu"
|
|
"csrc/quantization/cutlass_w8a8/scaled_mm_c3x.cu")
|
|
|
|
#
|
|
# The CUTLASS kernels for Hopper require sm90a to be enabled.
|
|
# This is done via the below gencode option, BUT that creates kernels for both sm90 and sm90a.
|
|
# That adds an extra 17MB to compiled binary, so instead we selectively enable it.
|
|
if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.0)
|
|
set_source_files_properties(
|
|
"csrc/quantization/cutlass_w8a8/scaled_mm_c3x.cu"
|
|
PROPERTIES
|
|
COMPILE_FLAGS
|
|
"-gencode arch=compute_90a,code=sm_90a")
|
|
endif()
|
|
|
|
#
|
|
# Machete kernels
|
|
|
|
# The machete kernels only work on hopper and require CUDA 12.0 or later.
|
|
if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER 12.0)
|
|
#
|
|
# For the Machete kernels we automatically generate sources for various
|
|
# preselected input type pairs and schedules.
|
|
# Generate sources:
|
|
execute_process(
|
|
COMMAND ${CMAKE_COMMAND} -E env
|
|
PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}/csrc/cutlass_extensions/:${CUTLASS_DIR}/python/:${VLLM_PYTHON_PATH}:$PYTHONPATH
|
|
${Python_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/csrc/quantization/machete/generate.py
|
|
RESULT_VARIABLE machete_generation_result
|
|
OUTPUT_VARIABLE machete_generation_output
|
|
OUTPUT_FILE ${CMAKE_CURRENT_BINARY_DIR}/machete_generation.log
|
|
ERROR_FILE ${CMAKE_CURRENT_BINARY_DIR}/machete_generation.log
|
|
)
|
|
|
|
if (NOT machete_generation_result EQUAL 0)
|
|
message(FATAL_ERROR "Machete generation failed."
|
|
" Result: \"${machete_generation_result}\""
|
|
"\nCheck the log for details: "
|
|
"${CMAKE_CURRENT_BINARY_DIR}/machete_generation.log")
|
|
else()
|
|
message(STATUS "Machete generation completed successfully.")
|
|
endif()
|
|
|
|
# Add machete generated sources
|
|
file(GLOB MACHETE_GEN_SOURCES "csrc/quantization/machete/generated/*.cu")
|
|
list(APPEND VLLM_EXT_SRC ${MACHETE_GEN_SOURCES})
|
|
message(STATUS "Machete generated sources: ${MACHETE_GEN_SOURCES}")
|
|
|
|
set_source_files_properties(
|
|
${MACHETE_GEN_SOURCES}
|
|
PROPERTIES
|
|
COMPILE_FLAGS
|
|
"-gencode arch=compute_90a,code=sm_90a")
|
|
endif()
|
|
|
|
# Add pytorch binding for machete (add on even CUDA < 12.0 so that we can
|
|
# raise an error if the user that this was built with an incompatible
|
|
# CUDA version)
|
|
list(APPEND VLLM_EXT_SRC
|
|
csrc/quantization/machete/machete_pytorch.cu)
|
|
endif()
|
|
|
|
define_gpu_extension_target(
|
|
_C
|
|
DESTINATION vllm
|
|
LANGUAGE ${VLLM_GPU_LANG}
|
|
SOURCES ${VLLM_EXT_SRC}
|
|
COMPILE_FLAGS ${VLLM_GPU_FLAGS}
|
|
ARCHITECTURES ${VLLM_GPU_ARCHES}
|
|
INCLUDE_DIRECTORIES ${CUTLASS_INCLUDE_DIR}
|
|
USE_SABI 3
|
|
WITH_SOABI)
|
|
|
|
#
|
|
# _moe_C extension
|
|
#
|
|
|
|
set(VLLM_MOE_EXT_SRC
|
|
"csrc/moe/torch_bindings.cpp"
|
|
"csrc/moe/topk_softmax_kernels.cu")
|
|
|
|
define_gpu_extension_target(
|
|
_moe_C
|
|
DESTINATION vllm
|
|
LANGUAGE ${VLLM_GPU_LANG}
|
|
SOURCES ${VLLM_MOE_EXT_SRC}
|
|
COMPILE_FLAGS ${VLLM_GPU_FLAGS}
|
|
ARCHITECTURES ${VLLM_GPU_ARCHES}
|
|
USE_SABI 3
|
|
WITH_SOABI)
|
|
|
|
|
|
|
|
if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
|
|
message(STATUS "Enabling C extension.")
|
|
add_dependencies(default _C)
|
|
|
|
message(STATUS "Enabling moe extension.")
|
|
add_dependencies(default _moe_C)
|
|
|
|
endif()
|