Nigel
/
jittor
forked from jittor/jittor
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

support builtin download of onednn because homebrew no longer provides onednn 2.x

This commit is contained in:
lzhengning 2023-01-05 09:38:03 +08:00
parent cc0e4f120a
commit 5a17e3d265
7 changed files with 89 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.cn.md
View File

@ -126,7 +126,7 @@ macOS 请使用 [homebrew](https://brew.sh) 安装额外的依赖。
```bash
brew install onednn libomp
brew install libomp
```
之后您可以通过 pip 安装 jittor，并测试是否可以成功运行。

4
README.md
View File

@ -122,7 +122,7 @@ python3.7 -m jittor.test.test_example
Please first install additional dependencies with [homebrew](https://brew.sh).
```bash
brew install onednn libomp
brew install libomp
```
@ -134,7 +134,7 @@ python3.7 -m jittor.test.test_example
```
Currently jittor only supports CPU in macOS.
Currently jittor only supports CPU on macOS.
### Windows install

4
README.src.md
View File

@ -158,7 +158,7 @@ macOS 请使用 [homebrew](https://brew.sh) 安装额外的依赖。
Please first install additional dependencies with [homebrew](https://brew.sh).
```bash
brew install onednn libomp
brew install libomp
```
之后您可以通过 pip 安装 jittor，并测试是否可以成功运行。
@ -172,7 +172,7 @@ python3.7 -m jittor.test.test_example
目前在 macOS 中，jittor 只支持 CPU 计算。
Currently jittor only supports CPU in macOS.
Currently jittor only supports CPU on macOS.
### Windows安装

35
python/jittor/compile_extern.py
View File

@ -51,6 +51,13 @@ def install_mkl(root_folder):
# url = "https://github.com/oneapi-src/oneDNN/releases/download/v2.2/dnnl_win_2.2.0_cpu_vcomp.zip"
filename = "dnnl_win_2.2.0_cpu_vcomp.zip"
md5 = "fa12c693b2ec07700d174e1e99d60a7e"
elif platform.system() == "Darwin":
if platform.machine() == "arm64":
filename = "dnnl_mac_2.2.0_cpu_omp_arm64.tgz"
md5 = "d8fdf56d3cf618685d22d18f08119f88"
else:
filename = "dnnl_mac_2.2.0_cpu_omp_x86_64"
md5 = "6e2f065d6a589c82081536b684768fe6"
else:
raise RuntimeError(f"platform.machine()=={platform.machine()} not support yet,"
" Please contact us on https://github.com/jittor/jittor ")
@ -61,7 +68,8 @@ def install_mkl(root_folder):
dirname = os.path.join(root_folder, filename.rsplit(".",1)[0])
if not (os.path.isfile(os.path.join(dirname, "lib", "libmkldnn.so")) or
os.path.isfile(os.path.join(dirname, "bin", "dnnl.dll"))):
os.path.isfile(os.path.join(dirname, "bin", "dnnl.dll")) or
os.path.isfile(os.path.join(dirname, "lib", "libmkldnn.dylib"))):
LOG.i("Downloading mkl...")
download_url_to_local(url, filename, root_folder, md5)
if fullname.endswith(".zip"):
@ -81,6 +89,9 @@ def install_mkl(root_folder):
assert 0 == os.system(cmd)
assert 0 == os.system(f"{dirname}/examples/test")
if platform.system() == "Darwin":
assert 0 == os.system(f"cd {dirname}/examples && "
f"{cc_path} -std=c++14 cnn_inference_f32.cpp -Ofast -lmkldnn -I ../include -L ../lib -o test && DYLD_LIBRARY_PATH=../lib/ ./test")
else:
assert 0 == os.system(f"cd {dirname}/examples && "
f"{cc_path} -std=c++14 cnn_inference_f32.cpp -Ofast -lmkldnn -I ../include -L ../lib -o test && LD_LIBRARY_PATH=../lib/ ./test")
@ -105,9 +116,7 @@ def setup_mkl():
mkl_include_path = os.environ.get("mkl_include_path")
mkl_lib_path = os.environ.get("mkl_lib_path")
if platform.system() == 'Linux' or os.name == 'nt':
if mkl_lib_path is None or mkl_include_path is None:
mkl_install_sh = os.path.join(jittor_path, "script", "install_mkl.sh")
LOG.v("setup mkl...")
# mkl_path = os.path.join(cache_path, "mkl")
# mkl_path decouple with cc_path
@ -130,6 +139,9 @@ def setup_mkl():
mkl_lib_name = os.path.join(mkl_home, 'bin', 'dnnl.dll')
mkl_bin_path = os.path.join(mkl_home, 'bin')
extra_flags = f" -I\"{mkl_include_path}\" -L\"{mkl_lib_path}\" -L\"{mkl_bin_path}\" -ldnnl "
elif platform.system() == "Darwin":
mkl_lib_name = os.path.join(mkl_lib_path, "libmkldnn.dylib")
assert os.path.isdir(mkl_include_path)
assert os.path.isdir(mkl_lib_path)
assert os.path.isfile(mkl_lib_name)
@ -139,15 +151,6 @@ def setup_mkl():
# We do not link manualy, link in custom ops
# ctypes.CDLL(mkl_lib_name, dlopen_flags)
elif platform.system() == 'Darwin':
mkl_lib_paths = [
"/usr/local/lib/libdnnl.dylib", # x86_64
"/opt/homebrew/lib/libdnnl.dylib", # arm64
]
if not any([os.path.exists(lib) for lib in mkl_lib_paths]):
raise RuntimeError("Not found onednn, please install it by the command 'brew install onednn'")
extra_flags = f" -ldnnl "
mkl_op_dir = os.path.join(jittor_path, "extern", "mkl", "ops")
mkl_op_files = [os.path.join(mkl_op_dir, name) for name in os.listdir(mkl_op_dir)]
mkl_ops = compile_custom_ops(mkl_op_files, extra_flags=extra_flags)
@ -585,10 +588,10 @@ setup_nccl()
setup_cutt()
try:
setup_mkl()
except Exception as e:
LOG.w("MKL install failed, msg:", e)
# try:
setup_mkl()
# except Exception as e:
# LOG.w("MKL install failed, msg:", e)
setup_cuda_extern()

17
python/jittor/compiler.py
View File

@ -1034,14 +1034,19 @@ cc_flags += " -fdiagnostics-color=always "
# 2. Non standard include path
if platform.system() == 'Darwin':
# TODO: if not using apple clang, there is no need to add -lomp
cc_flags += "-undefined dynamic_lookup -lomp "
if os.environ.get('CONDA_PREFIX', None):
cc_flags += " -undefined dynamic_lookup -lomp "
if os.environ.get("CONDA_PREFIX", None):
cc_flags += f" -L{os.path.join(os.environ['CONDA_PREFIX'], 'lib')} "
if platform.machine() == "arm64":
cc_flags += " -I/opt/homebrew/include -L/opt/homebrew/lib "
# if platform.machine() == "arm64":
# cc_flags += " -I/opt/homebrew/include -L/opt/homebrew/lib "
# Homebrew does not symlink the openmp library (libomp >= 15.0.6) into /opt/homebrew/lib
if os.path.exists('/opt/homebrew/opt/libomp'):
cc_flags += " -I/opt/homebrew/opt/libomp/include -L/opt/homebrew/opt/libomp/lib"
homebrew_openmp_paths = [
"/opt/homebrew/opt/libomp",
"/usr/local/opt/libomp"
]
for openmp_path in homebrew_openmp_paths:
if os.path.exists(openmp_path):
cc_flags += f" -I{openmp_path}/include -L{openmp_path}/lib"
# 3. User specified flags
if "cc_flags" in os.environ:

25
python/jittor_utils/__init__.py
View File

@ -460,16 +460,26 @@ def get_py3_config_path():
# python-config path only from sys.executable.
# To address this issue, we add predefined paths to search,
# - Linux: /usr/bin/python3.x-config
# - macOS (installed via homebrew): /usr/local/bin/python3.x-config
# - macOS:
# - shiped with macOS 13: /Library/Developer/CommandLineTools/Library/Frameworks/
# Python3.framework/Versions/3.x/lib/python3.x/config-3.x-darwin/python-config.py
# - installed via homebrew: /usr/local/bin/python3.x-config
# There may be issues under other cases, e.g., installed via conda.
py3_config_paths = [
os.path.dirname(sys.executable) + f"/python3.{sys.version_info.minor}-config",
sys.executable + "-config",
f"/usr/bin/python3.{sys.version_info.minor}-config",
f"/usr/local/bin/python3.{sys.version_info.minor}-config",
f'/opt/homebrew/bin/python3.{sys.version_info.minor}-config',
os.path.dirname(sys.executable) + "/python3-config",
]
if platform.system() == "Darwin":
if "homebrew" in sys.executable:
py3_config_paths.append(f'/opt/homebrew/bin/python3.{sys.version_info.minor}-config')
else:
py3_config_paths.append(f'/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/'\
f'Versions/3.{sys.version_info.minor}/lib/python3.{sys.version_info.minor}/'\
f'config-3.{sys.version_info.minor}-darwin/python-config.py')
if "python_config_path" in os.environ:
py3_config_paths.insert(0, os.environ["python_config_path"])
@ -498,6 +508,17 @@ def get_py3_include_path():
) + '"'
else:
_py3_include_path = run_cmd(get_py3_config_path()+" --includes")
# macOS (>=13) is shiped with a fake python3-config which outputs wrong include paths
# check the include paths and fix them
if platform.system() == "Darwin":
is_real_path = False
for include_path in _py3_include_path.strip().split():
if os.path.exists(include_path[2:]):
is_real_path = True
if not is_real_path:
_py3_include_path = f"-I/Library/Developer/CommandLineTools/Library/Frameworks/"\
f"Python3.framework/Versions/3.{sys.version_info.minor}/Headers"
return _py3_include_path

2
python/jittor_utils/misc.py
View File

@ -73,7 +73,7 @@ def download_url_to_local(url, filename, root_folder, md5):
os.remove(file_path)
raise RuntimeError(msg)
if not check_file_exist(file_path, md5):
raise RuntimeError("File downloads failed.")
raise RuntimeError(f"MD5 mismatch between the server and the downloaded file {file_path}")