forked from mindspore-Ecosystem/mindspore
Add modelarts export support for some network
This commit is contained in:
zhanghuiyao
parent
1983ded03f
commit
060cf9b17b
|
|
@ -392,6 +392,35 @@ run_standalone_train.sh
|
|||
# (7) Create your job.
|
||||
```
|
||||
|
||||
- Export on ModelArts (If you want to run in modelarts, please check the official documentation of [modelarts](https://support.huaweicloud.com/modelarts/), and you can start evaluating as follows)
|
||||
|
||||
1. Export s8 multiscale and flip with voc val dataset on modelarts, evaluating steps are as follows:
|
||||
|
||||
```python
|
||||
# (1) Perform a or b.
|
||||
# a. Set "enable_modelarts=True" on base_config.yaml file.
|
||||
# Set "export_model='deeplab_v3_s8'" on base_config.yaml file.
|
||||
# Set "export_batch_size=1" on base_config.yaml file.
|
||||
# Set "file_name='deeplabv3'" on base_config.yaml file.
|
||||
# Set "file_format='AIR'" on base_config.yaml file.
|
||||
# Set "checkpoint_url='/The path of checkpoint in S3/'" on beta_config.yaml file.
|
||||
# Set "ckpt_file='/cache/checkpoint_path/model.ckpt'" on base_config.yaml file.
|
||||
# Set other parameters on base_config.yaml file you need.
|
||||
# b. Add "enable_modelarts=True" on the website UI interface.
|
||||
# Add "export_model='deeplab_v3_s8'" on the website UI interface.
|
||||
# Add "export_batch_size=1" on the website UI interface.
|
||||
# Add "file_name='deeplabv3'" on the website UI interface.
|
||||
# Add "file_format='AIR'" on the website UI interface.
|
||||
# Add "checkpoint_url='/The path of checkpoint in S3/'" on the website UI interface.
|
||||
# Add "ckpt_file='/cache/checkpoint_path/model.ckpt'" on the website UI interface.
|
||||
# Add other parameters on the website UI interface.
|
||||
# (2) Upload or copy your trained model to S3 bucket.
|
||||
# (3) Set the code directory to "/path/deeplabv3" on the website UI interface.
|
||||
# (4) Set the startup file to "export.py" on the website UI interface.
|
||||
# (5) Set the "Dataset path" and "Output file path" and "Job log path" to your path on the website UI interface.
|
||||
# (6) Create your job.
|
||||
```
|
||||
|
||||
# [Script Description](#contents)
|
||||
|
||||
## [Script and Sample Code](#contents)
|
||||
|
|
|
|||
|
|
@ -407,6 +407,35 @@ run_standalone_train.sh
|
|||
# (7) 创建训练作业
|
||||
```
|
||||
|
||||
- 在 ModelArts 进行导出 (如果你想在modelarts上运行，可以参考以下文档 [modelarts](https://support.huaweicloud.com/modelarts/))
|
||||
|
||||
1. 使用voc val数据集评估多尺度和翻转s8。评估步骤如下：
|
||||
|
||||
```python
|
||||
# (1) 执行 a 或者 b.
|
||||
# a. 在 base_config.yaml 文件中设置 "enable_modelarts=True"
|
||||
# 在 base_config.yaml 文件中设置 "export_model='deeplab_v3_s8'"
|
||||
# 在 base_config.yaml 文件中设置 "export_batch_size=1"
|
||||
# 在 base_config.yaml 文件中设置 "file_name='deeplabv3'"
|
||||
# 在 base_config.yaml 文件中设置 "file_format='AIR'"
|
||||
# 在 base_config.yaml 文件中设置 "checkpoint_url='/The path of checkpoint in S3/'"
|
||||
# 在 base_config.yaml 文件中设置 "ckpt_file='/cache/checkpoint_path/model.ckpt'"
|
||||
# 在 base_config.yaml 文件中设置 其他参数
|
||||
# b. 在网页上设置 "enable_modelarts=True"
|
||||
# 在网页上设置 "export_model='deeplab_v3_s8'"
|
||||
# 在网页上设置 "export_batch_size=1"
|
||||
# 在网页上设置 "file_name='deeplabv3'"
|
||||
# 在网页上设置 "file_format='AIR'"
|
||||
# 在网页上设置 "checkpoint_url='/The path of checkpoint in S3/'"
|
||||
# 在网页上设置 "ckpt_file='/cache/checkpoint_path/model.ckpt'"
|
||||
# 在网页上设置 其他参数
|
||||
# (2) 上传你的预训练模型到 S3 桶上
|
||||
# (3) 在网页上设置你的代码路径为 "/path/deeplabv3"
|
||||
# (4) 在网页上设置启动文件为 "export.py"
|
||||
# (5) 在网页上设置"训练数据集"、"训练输出文件路径"、"作业日志路径"等
|
||||
# (6) 创建训练作业
|
||||
```
|
||||
|
||||
# 脚本说明
|
||||
|
||||
## 脚本及样例代码
|
||||
|
|
|
|||
|
|
@ -57,6 +57,15 @@ flip: False
|
|||
ckpt_path: ""
|
||||
input_format: "NCHW" # ["NCHW", "NHWC"]
|
||||
|
||||
# export param
|
||||
device_id: 0
|
||||
export_batch_size: 1
|
||||
input_size: 513
|
||||
ckpt_file: ""
|
||||
file_name: "deeplabv3"
|
||||
file_format: "AIR"
|
||||
export_model: "deeplab_v3_s8"
|
||||
|
||||
---
|
||||
|
||||
# Help description for each configuration
|
||||
|
|
@ -97,4 +106,13 @@ data_lst: "list of val data"
|
|||
scales: "scales of evaluation"
|
||||
flip: "perform left-right flip"
|
||||
ckpt_path: "model to evaluat"
|
||||
input_format: "NCHW or NHWC"
|
||||
input_format: "NCHW or NHWC"
|
||||
|
||||
# export param
|
||||
device_id: "Device id"
|
||||
export_batch_size: "batch size for export"
|
||||
input_size: "input_size"
|
||||
ckpt_file: "Checkpoint file path."
|
||||
file_name: "output file name."
|
||||
file_format: "file format, choices in ['AIR', 'MINDIR']"
|
||||
export_model: "Select model structure (Default: deeplab_v3_s8), choices in ['deeplab_v3_s16', 'deeplab_v3_s8']"
|
||||
|
|
@ -13,7 +13,7 @@
|
|||
# limitations under the License.
|
||||
# ============================================================================
|
||||
"""export checkpoint file into air, onnx, mindir models"""
|
||||
import argparse
|
||||
import os
|
||||
import numpy as np
|
||||
|
||||
import mindspore.nn as nn
|
||||
|
|
@ -21,25 +21,8 @@ import mindspore.ops as ops
|
|||
from mindspore import Tensor, context, load_checkpoint, load_param_into_net, export
|
||||
from src.nets import net_factory
|
||||
|
||||
parser = argparse.ArgumentParser(description='checkpoint export')
|
||||
parser.add_argument("--device_id", type=int, default=0, help="Device id")
|
||||
parser.add_argument("--batch_size", type=int, default=1, help="batch size")
|
||||
parser.add_argument("--input_size", type=int, default=513, help="batch size")
|
||||
parser.add_argument("--ckpt_file", type=str, required=True, help="Checkpoint file path.")
|
||||
parser.add_argument("--file_name", type=str, default="deeplabv3", help="output file name.")
|
||||
parser.add_argument('--file_format', type=str, choices=["AIR", "MINDIR"], default='AIR', help='file format')
|
||||
parser.add_argument("--device_target", type=str, choices=["Ascend", "GPU", "CPU"], default="Ascend",
|
||||
help="device target")
|
||||
parser.add_argument('--model', type=str.lower, default='deeplab_v3_s8', choices=['deeplab_v3_s16', 'deeplab_v3_s8'],
|
||||
help='Select model structure (Default: deeplab_v3_s8)')
|
||||
parser.add_argument('--num_classes', type=int, default=21, help='the number of classes (Default: 21)')
|
||||
parser.add_argument("--input_format", type=str, choices=["NCHW", "NHWC"], default="NCHW",
|
||||
help="NCHW or NHWC")
|
||||
args = parser.parse_args()
|
||||
|
||||
context.set_context(mode=context.GRAPH_MODE, device_target=args.device_target)
|
||||
if args.device_target == "Ascend":
|
||||
context.set_context(device_id=args.device_id)
|
||||
from model_utils.config import config
|
||||
from model_utils.moxing_adapter import moxing_wrapper
|
||||
|
||||
class BuildEvalNetwork(nn.Cell):
|
||||
def __init__(self, net, input_format="NCHW"):
|
||||
|
|
@ -56,18 +39,36 @@ class BuildEvalNetwork(nn.Cell):
|
|||
output = self.softmax(output)
|
||||
return output
|
||||
|
||||
if __name__ == '__main__':
|
||||
if args.model == 'deeplab_v3_s16':
|
||||
network = net_factory.nets_map['deeplab_v3_s16']('eval', args.num_classes, 16, True)
|
||||
|
||||
def modelarts_pre_process():
|
||||
'''modelarts pre process function.'''
|
||||
config.file_name = os.path.join(config.output_path, config.file_name)
|
||||
|
||||
|
||||
@moxing_wrapper(pre_process=modelarts_pre_process)
|
||||
def run_export():
|
||||
'''run export.'''
|
||||
context.set_context(mode=context.GRAPH_MODE, device_target=config.device_target)
|
||||
if config.device_target == "Ascend":
|
||||
context.set_context(device_id=config.device_id)
|
||||
|
||||
if config.export_model == 'deeplab_v3_s16':
|
||||
network = net_factory.nets_map['deeplab_v3_s16']('eval', config.num_classes, 16, True)
|
||||
else:
|
||||
network = net_factory.nets_map['deeplab_v3_s8']('eval', args.num_classes, 8, True)
|
||||
network = BuildEvalNetwork(network, args.input_format)
|
||||
param_dict = load_checkpoint(args.ckpt_file)
|
||||
network = net_factory.nets_map['deeplab_v3_s8']('eval', config.num_classes, 8, True)
|
||||
network = BuildEvalNetwork(network, config.input_format)
|
||||
param_dict = load_checkpoint(config.ckpt_file)
|
||||
|
||||
# load the parameter into net
|
||||
load_param_into_net(network, param_dict)
|
||||
if args.input_format == "NHWC":
|
||||
input_data = Tensor(np.ones([args.batch_size, args.input_size, args.input_size, 3]).astype(np.float32))
|
||||
if config.input_format == "NHWC":
|
||||
input_data = Tensor(
|
||||
np.ones([config.export_batch_size, config.input_size, config.input_size, 3]).astype(np.float32))
|
||||
else:
|
||||
input_data = Tensor(np.ones([args.batch_size, 3, args.input_size, args.input_size]).astype(np.float32))
|
||||
export(network, input_data, file_name=args.file_name, file_format=args.file_format)
|
||||
input_data = Tensor(
|
||||
np.ones([config.export_batch_size, 3, config.input_size, config.input_size]).astype(np.float32))
|
||||
export(network, input_data, file_name=config.file_name, file_format=config.file_format)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run_export()
|
||||
|
|
|
|||
|
|
@ -211,6 +211,25 @@ sh run_eval_ascend.sh [IMGS_PATH] [ANNOS_PATH] [CHECKPOINT_PATH] [COCO_TEXT_PARS
|
|||
# (5) Set the startup file to "eval.py" on the website UI interface.
|
||||
# (6) Set the "Dataset path" and "Output file path" and "Job log path" to your path on the website UI interface.
|
||||
# (7) Create your job.
|
||||
#
|
||||
# Export 1p on ModelArts
|
||||
# (1) Perform a or b.
|
||||
# a. Set "enable_modelarts=True" on default_config.yaml file.
|
||||
# Set "checkpoint_url='s3://dir_to_trained_model/'" on default_config.yaml file.
|
||||
# Set "ckpt_file='/cache/checkpoint_path/model.ckpt'" on default_config.yaml file.
|
||||
# Set "device_target='Ascend'" on default_config.yaml file.
|
||||
# Set "file_format='MINDIR'" on default_config.yaml file.
|
||||
# Set other parameters on default_config.yaml file you need.
|
||||
# b. Add "enable_modelarts=True" on the website UI interface.
|
||||
# Add "checkpoint_url='s3://dir_to_trained_model/'" on the website UI interface.
|
||||
# Add "ckpt_file='/cache/checkpoint_path/model.ckpt'" on the website UI interface.
|
||||
# Add "device_target='Ascend'" on the website UI interface.
|
||||
# Add "file_format='MINDIR'" on the website UI interface.
|
||||
# Add other parameters on the website UI interface.
|
||||
# (2) Set the code directory to "/path/deeptext" on the website UI interface.
|
||||
# (3) Set the startup file to "export.py" on the website UI interface.
|
||||
# (4) Set the "Dataset path" and "Output file path" and "Job log path" to your path on the website UI interface.
|
||||
# (5) Create your job.
|
||||
```
|
||||
|
||||
### Launch
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@
|
|||
# limitations under the License.
|
||||
# ============================================================================
|
||||
"""export checkpoint file into air, mindir models"""
|
||||
import os
|
||||
import numpy as np
|
||||
|
||||
import mindspore as ms
|
||||
|
|
@ -22,14 +23,23 @@ from mindspore import Tensor, load_checkpoint, load_param_into_net, export, cont
|
|||
from src.Deeptext.deeptext_vgg16 import Deeptext_VGG16_Infer
|
||||
|
||||
from model_utils.config import config
|
||||
from model_utils.moxing_adapter import moxing_wrapper
|
||||
from model_utils.device_adapter import get_device_id
|
||||
|
||||
|
||||
config.test_batch_size = config.export_batch_size
|
||||
context.set_context(mode=context.GRAPH_MODE, device_target=config.export_device_target)
|
||||
context.set_context(device_id=get_device_id())
|
||||
|
||||
if __name__ == '__main__':
|
||||
def modelarts_pre_process():
|
||||
'''modelarts pre process function.'''
|
||||
config.file_name = os.path.join(config.output_path, config.file_name)
|
||||
|
||||
|
||||
@moxing_wrapper(pre_process=modelarts_pre_process)
|
||||
def run_export():
|
||||
'''run export.'''
|
||||
config.test_batch_size = config.export_batch_size
|
||||
context.set_context(mode=context.GRAPH_MODE, device_target=config.export_device_target)
|
||||
context.set_context(device_id=get_device_id())
|
||||
|
||||
net = Deeptext_VGG16_Infer(config=config)
|
||||
net.set_train(False)
|
||||
|
||||
|
|
@ -48,3 +58,7 @@ if __name__ == '__main__':
|
|||
img_data = Tensor(np.zeros([config.test_batch_size, 3, config.img_height, config.img_width]), ms.float32)
|
||||
|
||||
export(net, img_data, file_name=config.file_name, file_format=config.file_format)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run_export()
|
||||
|
|
|
|||
|
|
@ -204,7 +204,7 @@ We use CIFAR-10 dataset by default. Your can also pass `$dataset_type` to the sc
|
|||
# (7) Create your job.
|
||||
```
|
||||
|
||||
- Eval imagenet on ModelArts
|
||||
- Eval cifar10 on ModelArts
|
||||
|
||||
```python
|
||||
# (1) Add "config_path='/path_to_code/cifar10_config.yaml'" on the website UI interface.
|
||||
|
|
@ -229,6 +229,26 @@ We use CIFAR-10 dataset by default. Your can also pass `$dataset_type` to the sc
|
|||
# (8) Create your job.
|
||||
```
|
||||
|
||||
- Export on ModelArts
|
||||
|
||||
```python
|
||||
# (1) Add "config_path='/path_to_code/cifar10_config.yaml'" on the website UI interface.
|
||||
# (2) Perform a or b.
|
||||
# a. Set "enable_modelarts=True" on cifar10_config.yaml file.
|
||||
# Set "checkpoint_url='s3://dir_to_trained_ckpt/'" on cifar10_config.yaml file.
|
||||
# Set "ckpt_file='/cache/checkpoint_path/model.ckpt'" on cifar10_config.yaml file.
|
||||
# Set other parameters on cifar10_config.yaml file you need.
|
||||
# b. Add "enable_modelarts=True" on the website UI interface.
|
||||
# Add "checkpoint_url=s3://dir_to_trained_ckpt/" on the website UI interface.
|
||||
# Add "ckpt_file=/cache/checkpoint_path/model.ckpt" on the website UI interface.
|
||||
# Add other parameters on the website UI interface.
|
||||
# (3) Upload or copy your trained model to S3 bucket.
|
||||
# (4) Set the code directory to "/path/googlenet" on the website UI interface.
|
||||
# (5) Set the startup file to "export.py" on the website UI interface.
|
||||
# (6) Set the "Dataset path" and "Output file path" and "Job log path" to your path on the website UI interface.
|
||||
# (7) Create your job.
|
||||
```
|
||||
|
||||
# [Script Description](#contents)
|
||||
|
||||
## [Script and Sample Code](#contents)
|
||||
|
|
|
|||
|
|
@ -236,6 +236,26 @@ GoogleNet由多个inception模块串联起来，可以更加深入。 降维的
|
|||
# (8) 创建训练作业
|
||||
```
|
||||
|
||||
- 在 ModelArts 上使用单卡导出 cifar10 数据集
|
||||
|
||||
```python
|
||||
# (1) 在网页上设置 "config_path='/path_to_code/cifar10_config.yaml'"
|
||||
# (2) 执行a或者b
|
||||
# a. 在 cifar10_config.yaml 文件中设置 "enable_modelarts=True"
|
||||
# 在 cifar10_config.yaml 文件中设置 "checkpoint_url='s3://dir_to_trained_ckpt/'"
|
||||
# 在 cifar10_config.yaml 文件中设置 "ckpt_file='/cache/checkpoint_path/model.ckpt'"
|
||||
# 在 cifar10_config.yaml 文件中设置 其他参数
|
||||
# b. 在网页上设置 "enable_modelarts=True"
|
||||
# 在网页上设置 "checkpoint_url=s3://dir_to_trained_ckpt/" on the website UI interface.
|
||||
# 在网页上设置 "ckpt_file=/cache/checkpoint_path/model.ckpt" on the website UI interface.
|
||||
# 在网页上设置 其他参数
|
||||
# (3) 上传你的预训练模型到 S3 桶上
|
||||
# (5) 在网页上设置你的代码路径为 "/path/googlenet"
|
||||
# (6) 在网页上设置启动文件为 "export.py"
|
||||
# (7) 在网页上设置"训练数据集"、"训练输出文件路径"、"作业日志路径"等
|
||||
# (8) 创建训练作业
|
||||
```
|
||||
|
||||
# 脚本说明
|
||||
|
||||
## 脚本及样例代码
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@
|
|||
##############export checkpoint file into air, mindir models#################
|
||||
python export.py
|
||||
"""
|
||||
import os
|
||||
import numpy as np
|
||||
|
||||
import mindspore as ms
|
||||
|
|
@ -24,13 +25,21 @@ from mindspore import Tensor, load_checkpoint, load_param_into_net, export, cont
|
|||
from src.googlenet import GoogleNet
|
||||
|
||||
from model_utils.config import config
|
||||
from model_utils.moxing_adapter import moxing_wrapper
|
||||
from model_utils.device_adapter import get_device_id
|
||||
|
||||
context.set_context(mode=context.GRAPH_MODE, device_target=config.device_target)
|
||||
if config.device_target == "Ascend":
|
||||
context.set_context(device_id=get_device_id())
|
||||
|
||||
if __name__ == '__main__':
|
||||
def modelarts_pre_process():
|
||||
'''modelarts pre process function.'''
|
||||
config.file_name = os.path.join(config.output_path, config.file_name)
|
||||
|
||||
|
||||
@moxing_wrapper(pre_process=modelarts_pre_process)
|
||||
def run_export():
|
||||
context.set_context(mode=context.GRAPH_MODE, device_target=config.device_target)
|
||||
if config.device_target == "Ascend":
|
||||
context.set_context(device_id=get_device_id())
|
||||
|
||||
net = GoogleNet(num_classes=config.num_classes)
|
||||
|
||||
assert config.ckpt_file is not None, "config.ckpt_file is None."
|
||||
|
|
@ -39,3 +48,7 @@ if __name__ == '__main__':
|
|||
|
||||
input_arr = Tensor(np.ones([config.batch_size, 3, config.image_height, config.image_width]), ms.float32)
|
||||
export(net, input_arr, file_name=config.file_name, file_format=config.file_format)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run_export()
|
||||
|
|
|
|||
|
|
@ -229,12 +229,34 @@ python eval.py --config_path=[YAML_CONFIG_PATH] --device_target="GPU" --dataset=
|
|||
# Add "checkpoint_url=s3://dir_to_your_trained_model/" on the website UI interface.
|
||||
# Add "pre_trained=/cache/checkpoint_path/model.ckpt" on the website UI interface.
|
||||
# Add other parameters on the website UI interface.
|
||||
# (3) Upload or copy your pretrained model to S3 bucket.
|
||||
# (3) Upload or copy your trained model to S3 bucket.
|
||||
# (4) Upload a zip dataset to S3 bucket. (you could also upload the origin dataset, but it can be so slow.)
|
||||
# (5) Set the code directory to "/path/vgg16" on the website UI interface.
|
||||
# (6) Set the startup file to "eval.py" on the website UI interface.
|
||||
# (7) Set the "Dataset path" and "Output file path" and "Job log path" to your path on the website UI interface.
|
||||
# (8) Create your job.
|
||||
#
|
||||
# Export 1p on ModelArts
|
||||
# (1) Add "config_path=/path_to_code/imagenet2012_config.yaml" on the website UI interface.
|
||||
# (2) Perform a or b.
|
||||
# a. Set "enable_modelarts=True" on imagenet2012_config.yaml file.
|
||||
# Set "file_name='vgg16'" on imagenet2012_config.yaml file.
|
||||
# Set "file_format='AIR'" on imagenet2012_config.yaml file.
|
||||
# Set "checkpoint_url='s3://dir_to_your_trained_model/'" on imagenet2012_config.yaml file.
|
||||
# Set "ckpt_file='/cache/checkpoint_path/model.ckpt'" on imagenet2012_config.yaml file.
|
||||
# Set other parameters on imagenet2012_config.yaml file you need.
|
||||
# b. Add "enable_modelarts=True" on the website UI interface.
|
||||
# Add "file_name=vgg16" on the website UI interface.
|
||||
# Add "file_format=AIR" on the website UI interface.
|
||||
# Add "checkpoint_url=s3://dir_to_your_trained_model/" on the website UI interface.
|
||||
# Add "ckpt_file=/cache/checkpoint_path/model.ckpt" on the website UI interface.
|
||||
# Add other parameters on the website UI interface.
|
||||
# (3) Upload or copy your trained model to S3 bucket.
|
||||
# (4) Upload a zip dataset to S3 bucket. (you could also upload the origin dataset, but it can be so slow.)
|
||||
# (5) Set the code directory to "/path/vgg16" on the website UI interface.
|
||||
# (6) Set the startup file to "export.py" on the website UI interface.
|
||||
# (7) Set the "Dataset path" and "Output file path" and "Job log path" to your path on the website UI interface.
|
||||
# (8) Create your job.
|
||||
```
|
||||
|
||||
## [Script Description](#contents)
|
||||
|
|
|
|||
|
|
@ -238,6 +238,28 @@ python eval.py --config_path=[YAML_CONFIG_PATH] --device_target="GPU" --dataset=
|
|||
# (6) 在网页上设置启动文件为 "eval.py"
|
||||
# (7) 在网页上设置"训练数据集"、"训练输出文件路径"、"作业日志路径"等
|
||||
# (8) 创建训练作业
|
||||
#
|
||||
# 在 ModelArts 上使用 单卡导出
|
||||
# (1) 在网页上设置 "config_path=/path_to_code/imagenet2012_config.yaml"
|
||||
# (2) 执行a或者b
|
||||
# a. 在 imagenet2012_config.yaml 文件中设置 "enable_modelarts=True"
|
||||
# 在 imagenet2012_config.yaml 文件中设置 "file_name='vgg16'"
|
||||
# 在 imagenet2012_config.yaml 文件中设置 "file_format='AIR'"
|
||||
# 在 imagenet2012_config.yaml 文件中设置 "checkpoint_url='s3://dir_to_your_trained_model/'"
|
||||
# 在 imagenet2012_config.yaml 文件中设置 "ckpt_file='/cache/checkpoint_path/model.ckpt'"
|
||||
# 在 imagenet2012_config.yaml 文件中设置 其他参数
|
||||
# b. 在网页上设置 "enable_modelarts=True"
|
||||
# 在网页上设置 "file_name=vgg16"
|
||||
# 在网页上设置 "file_format=AIR"
|
||||
# 在网页上设置 "checkpoint_url=s3://dir_to_your_trained_model/"
|
||||
# 在网页上设置 "ckpt_file=/cache/checkpoint_path/model.ckpt"
|
||||
# 在网页上设置 其他参数
|
||||
# (3) 上传你的预训练模型到 S3 桶上
|
||||
# (4) 上传你的压缩数据集到 S3 桶上 (你也可以上传原始的数据集，但那可能会很慢。)
|
||||
# (5) 在网页上设置你的代码路径为 "/path/vgg16"
|
||||
# (6) 在网页上设置启动文件为 "eval.py"
|
||||
# (7) 在网页上设置"训练数据集"、"训练输出文件路径"、"作业日志路径"等
|
||||
# (8) 创建训练作业
|
||||
```
|
||||
|
||||
## 脚本说明
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@
|
|||
# limitations under the License.
|
||||
# ============================================================================
|
||||
"""export checkpoint file into models"""
|
||||
import os
|
||||
import numpy as np
|
||||
|
||||
from mindspore import Tensor, context
|
||||
|
|
@ -22,9 +23,16 @@ from mindspore.train.serialization import load_checkpoint, export
|
|||
from src.vgg import vgg16
|
||||
|
||||
from model_utils.moxing_adapter import config
|
||||
from model_utils.moxing_adapter import moxing_wrapper
|
||||
from model_utils.device_adapter import get_device_id
|
||||
|
||||
|
||||
def modelarts_pre_process():
|
||||
'''modelarts pre process function.'''
|
||||
config.file_name = os.path.join(config.output_path, config.file_name)
|
||||
|
||||
|
||||
@moxing_wrapper(pre_process=modelarts_pre_process)
|
||||
def run_export():
|
||||
config.image_size = list(map(int, config.image_size.split(',')))
|
||||
|
||||
|
|
|
|||
|
|
@ -170,6 +170,31 @@ After installing MindSpore via the official website and Dataset is correctly gen
|
|||
# (7) Create your job.
|
||||
```
|
||||
|
||||
- Export 1p on ModelArts Ascend/GPU
|
||||
|
||||
```python
|
||||
# (1) Perform a or b.
|
||||
# a. Set "enable_modelarts=True" on default_config.yaml file.
|
||||
# Set "ckpt_file='/cache/checkpoint_path/model.ckpt'" on default_config.yaml file.
|
||||
# Set "checkpoint_url='s3://dir_to_your_trained_ckpt/'" on default_config.yaml file.
|
||||
# Set "file_name='bgcf'" on default_config.yaml file.
|
||||
# Set "file_format='AIR'" on default_config.yaml file.
|
||||
# (options)Set "device_target='GPU'" on default_config.yaml file if run on GPU.
|
||||
# Set other parameters on default_config.yaml file you need.
|
||||
# b. Add "enable_modelarts=True" on the website UI interface.
|
||||
# Add "ckpt_file=/cache/checkpoint_path/model.ckpt" on the website UI interface.
|
||||
# Add "checkpoint_url=s3://dir_to_your_trained_ckpt/" on the website UI interface.
|
||||
# Add "file_name=bgcf" on the website UI interface.
|
||||
# Add "file_format=AIR" on the website UI interface.
|
||||
# (options)Add "device_target=GPU" on the website UI interface if run on GPU.
|
||||
# Add other parameters on the website UI interface.
|
||||
# (2) Upload or copy your trained model to S3 bucket.
|
||||
# (3) Set the code directory to "/path/bgcf" on the website UI interface.
|
||||
# (4) Set the startup file to "export.py" on the website UI interface.
|
||||
# (5) Set the "Dataset path" and "Output file path" and "Job log path" to your path on the website UI interface.
|
||||
# (6) Create your job.
|
||||
```
|
||||
|
||||
## [Script Description](#contents)
|
||||
|
||||
### [Script and Sample Code](#contents)
|
||||
|
|
|
|||
|
|
@ -149,7 +149,7 @@ BGCF包含两个主要模块。首先是抽样，它生成基于节点复制的
|
|||
# 在网页上设置 其他参数
|
||||
# (2) 在本地准备转换好的数据集并将其压缩为一个文件，如："amazon_beauty.zip" (数据集转换代码可以参考上面的Dataset章节)
|
||||
# (3) 上传你的压缩数据集到 S3 桶上 (你也可以上传原始的数据集，但那可能会很慢。)
|
||||
# (4) 在网页上设置你的代码路径为 "/path/googlenet"
|
||||
# (4) 在网页上设置你的代码路径为 "/path/bgcf"
|
||||
# (5) 在网页上设置启动文件为 "train.py"
|
||||
# (6) 在网页上设置"训练数据集"、"训练输出文件路径"、"作业日志路径"等
|
||||
# (7) 创建训练作业
|
||||
|
|
@ -175,12 +175,37 @@ BGCF包含两个主要模块。首先是抽样，它生成基于节点复制的
|
|||
# 在网页上设置 其他参数
|
||||
# (2) 在本地准备转换好的数据集并将其压缩为一个文件，如："amazon_beauty.zip" (数据集转换代码可以参考上面的Dataset章节)
|
||||
# (3) 上传你的压缩数据集到 S3 桶上 (你也可以上传原始的数据集，但那可能会很慢。)
|
||||
# (4) 在网页上设置你的代码路径为 "/path/googlenet"
|
||||
# (4) 在网页上设置你的代码路径为 "/path/bgcf"
|
||||
# (5) 在网页上设置启动文件为 "eval.py"
|
||||
# (6) 在网页上设置"训练数据集"、"训练输出文件路径"、"作业日志路径"等
|
||||
# (7) 创建训练作业
|
||||
```
|
||||
|
||||
- 在 ModelArts 上使用单卡导出（GPU or Ascend）
|
||||
|
||||
```python
|
||||
# (1) 执行a或者b
|
||||
# a. 在 default_config.yaml 文件中设置 "enable_modelarts=True"
|
||||
# 在 default_config.yaml 文件中设置 "ckpt_file='/cache/checkpoint_path/model.ckpt'"
|
||||
# 在 default_config.yaml 文件中设置 "checkpoint_url='s3://dir_to_your_trained_ckpt/'"
|
||||
# 在 default_config.yaml 文件中设置 "file_name='bgcf'"
|
||||
# 在 default_config.yaml 文件中设置 "file_format='AIR'"
|
||||
# (可选)在 default_config.yaml 文件中设置 "device_target='GPU'"
|
||||
# 在 default_config.yaml 文件中设置 其他参数
|
||||
# b. 在网页上设置 "enable_modelarts=True"
|
||||
# 在网页上设置 "ckpt_file=/cache/checkpoint_path/model.ckpt"
|
||||
# 在网页上设置 "checkpoint_url=s3://dir_to_your_trained_ckpt/"
|
||||
# 在网页上设置 "file_name=bgcf"
|
||||
# 在网页上设置 "file_format=AIR"
|
||||
# (可选)Add "device_target=GPU"
|
||||
# 在网页上设置 其他参数
|
||||
# (2) 上传你的预训练模型到 S3 桶上
|
||||
# (3) 在网页上设置你的代码路径为 "/path/bgcf"
|
||||
# (4) 在网页上设置启动文件为 "export.py"
|
||||
# (5) 在网页上设置"训练数据集"、"训练输出文件路径"、"作业日志路径"等
|
||||
# (6) 创建训练作业
|
||||
```
|
||||
|
||||
## 脚本说明
|
||||
|
||||
### 脚本及样例代码
|
||||
|
|
|
|||
|
|
@ -35,6 +35,13 @@ gnew_neighs: 20
|
|||
embedded_dimension: 64
|
||||
dist_reg: 0.003
|
||||
|
||||
# export option
|
||||
device_id: 0
|
||||
ckpt_file: ""
|
||||
file_name: "bgcf"
|
||||
file_format: "AIR"
|
||||
row_neighs: 40
|
||||
|
||||
---
|
||||
|
||||
# Help description for each configuration
|
||||
|
|
@ -58,4 +65,11 @@ raw_neighs: "num of sampling neighbors in raw graph"
|
|||
gnew_neighs: "num of sampling neighbors in sample graph"
|
||||
embedded_dimension: "output embedding dim"
|
||||
dist_reg: "distance loss coefficient"
|
||||
device_target: "device target, choices in ['Ascend', GPU]"
|
||||
device_target: "device target, choices in ['Ascend', GPU]"
|
||||
|
||||
# export option
|
||||
device_id: "Device id"
|
||||
ckpt_file: "Checkpoint file path."
|
||||
file_name: "output file name."
|
||||
file_format: "file format, choices in ['AIR', 'ONNX', 'MINDIR']"
|
||||
row_neighs: "num of sampling neighbors in raw graph"
|
||||
|
|
@ -13,7 +13,7 @@
|
|||
# limitations under the License.
|
||||
# ============================================================================
|
||||
"""export ckpt to model"""
|
||||
import argparse
|
||||
import os
|
||||
import numpy as np
|
||||
|
||||
from mindspore import context, Tensor
|
||||
|
|
@ -22,46 +22,47 @@ from mindspore.train.serialization import export, load_checkpoint
|
|||
from src.bgcf import BGCF
|
||||
from src.callback import ForwardBGCF
|
||||
|
||||
parser = argparse.ArgumentParser(description="bgcf export")
|
||||
parser.add_argument("--device_id", type=int, default=0, help="Device id")
|
||||
parser.add_argument("--ckpt_file", type=str, required=True, help="Checkpoint file path.")
|
||||
parser.add_argument("--file_name", type=str, default="bgcf", help="output file name.")
|
||||
parser.add_argument("--file_format", type=str, choices=["AIR", "ONNX", "MINDIR"], default="AIR", help="file format")
|
||||
parser.add_argument("--device_target", type=str, choices=["Ascend", "GPU", "CPU"], default="Ascend",
|
||||
help="device target")
|
||||
parser.add_argument("--input_dim", type=int, choices=[64, 128], default=64, help="embedding dimension")
|
||||
parser.add_argument("--embedded_dimension", type=int, default=64, help="output embedding dimension")
|
||||
parser.add_argument("--row_neighs", type=int, default=40, help="num of sampling neighbors in raw graph")
|
||||
parser.add_argument("--gnew_neighs", type=int, default=20, help="num of sampling neighbors in sample graph")
|
||||
parser.add_argument("--activation", type=str, default="tanh", choices=["relu", "tanh"], help="activation function")
|
||||
args = parser.parse_args()
|
||||
from model_utils.config import config
|
||||
from model_utils.moxing_adapter import moxing_wrapper
|
||||
|
||||
context.set_context(mode=context.GRAPH_MODE, device_target=args.device_target)
|
||||
if args.device_target == "Ascend":
|
||||
context.set_context(device_id=args.device_id)
|
||||
|
||||
if __name__ == "__main__":
|
||||
def modelarts_pre_process():
|
||||
'''modelarts pre process function.'''
|
||||
config.file_name = os.path.join(config.output_path, config.file_name)
|
||||
|
||||
|
||||
@moxing_wrapper(pre_process=modelarts_pre_process)
|
||||
def run_export():
|
||||
'''run export.'''
|
||||
context.set_context(mode=context.GRAPH_MODE, device_target=config.device_target)
|
||||
if config.device_target == "Ascend":
|
||||
context.set_context(device_id=config.device_id)
|
||||
|
||||
num_user, num_item = 7068, 3570
|
||||
|
||||
network = BGCF([args.input_dim, num_user, num_item],
|
||||
args.embedded_dimension,
|
||||
args.activation,
|
||||
network = BGCF([config.input_dim, num_user, num_item],
|
||||
config.embedded_dimension,
|
||||
config.activation,
|
||||
[0.0, 0.0, 0.0],
|
||||
num_user,
|
||||
num_item,
|
||||
args.input_dim)
|
||||
config.input_dim)
|
||||
|
||||
load_checkpoint(args.ckpt_file, net=network)
|
||||
load_checkpoint(config.ckpt_file, net=network)
|
||||
|
||||
forward_net = ForwardBGCF(network)
|
||||
|
||||
users = Tensor(np.zeros([num_user,]).astype(np.int32))
|
||||
items = Tensor(np.zeros([num_item,]).astype(np.int32))
|
||||
neg_items = Tensor(np.zeros([num_item, 1]).astype(np.int32))
|
||||
u_test_neighs = Tensor(np.zeros([num_user, args.row_neighs]).astype(np.int32))
|
||||
u_test_gnew_neighs = Tensor(np.zeros([num_user, args.gnew_neighs]).astype(np.int32))
|
||||
i_test_neighs = Tensor(np.zeros([num_item, args.row_neighs]).astype(np.int32))
|
||||
i_test_gnew_neighs = Tensor(np.zeros([num_item, args.gnew_neighs]).astype(np.int32))
|
||||
u_test_neighs = Tensor(np.zeros([num_user, config.row_neighs]).astype(np.int32))
|
||||
u_test_gnew_neighs = Tensor(np.zeros([num_user, config.gnew_neighs]).astype(np.int32))
|
||||
i_test_neighs = Tensor(np.zeros([num_item, config.row_neighs]).astype(np.int32))
|
||||
i_test_gnew_neighs = Tensor(np.zeros([num_item, config.gnew_neighs]).astype(np.int32))
|
||||
|
||||
input_data = [users, items, neg_items, u_test_neighs, u_test_gnew_neighs, i_test_neighs, i_test_gnew_neighs]
|
||||
export(forward_net, *input_data, file_name=args.file_name, file_format=args.file_format)
|
||||
export(forward_net, *input_data, file_name=config.file_name, file_format=config.file_format)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run_export()
|
||||
|
|
|
|||
|
|
@ -116,10 +116,28 @@ After dataset preparation, you can start training and evaluation as follows:
|
|||
# Add "model_ckpt='/cache/checkpoint_path/model.ckpt'" on the website UI interface.
|
||||
# (option)Set "device_target='GPU'" on the website UI interface if run with GPU.
|
||||
# (option)Set other parameters on the website UI interface.
|
||||
# (3) Upload or copy your pretrained model to S3 bucket.
|
||||
# (3) Upload or copy your trained model to S3 bucket.
|
||||
# (4) Upload a zip dataset to S3 bucket. (you could also upload the origin dataset, but it can be so slow.)
|
||||
# (5) Set the code directory to "/path/fasttext" on the website UI interface.
|
||||
# (6) Set the startup file to "train.py" on the website UI interface.
|
||||
# (6) Set the startup file to "eval.py" on the website UI interface.
|
||||
# (7) Set the "Dataset path" and "Output file path" and "Job log path" to your path on the website UI interface.
|
||||
# (8) Create your job.
|
||||
#
|
||||
# run export example
|
||||
# (1) Add "config_path='/path_to_code/[DATASET_NAME]_config.yaml'" on the website UI interface.
|
||||
# (2) Perform a or b.
|
||||
# a. Set "enable_modelarts=True" on [DATASET_NAME]_config.yaml file.
|
||||
# Set "checkpoint_url='s3://dir_to_trained_ckpt/'" on [DATASET_NAME]_config.yaml file.
|
||||
# Set "ckpt_file='/cache/checkpoint_path/model.ckpt'" on [DATASET_NAME]_config.yaml file.
|
||||
# Set other parameters on [DATASET_NAME]_config.yaml file you need.
|
||||
# b. Add "enable_modelarts=True" on the website UI interface.
|
||||
# Add "checkpoint_url='s3://dir_to_trained_ckpt/'" on the website UI interface.
|
||||
# Add "ckpt_file='/cache/checkpoint_path/model.ckpt'" on the website UI interface.
|
||||
# Add other parameters on the website UI interface.
|
||||
# (3) Upload or copy your trained model to S3 bucket.
|
||||
# (4) Upload a zip dataset to S3 bucket. (you could also upload the origin dataset, but it can be so slow.)
|
||||
# (5) Set the code directory to "/path/fasttext" on the website UI interface.
|
||||
# (6) Set the startup file to "export.py" on the website UI interface.
|
||||
# (7) Set the "Dataset path" and "Output file path" and "Job log path" to your path on the website UI interface.
|
||||
# (8) Create your job.
|
||||
```
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@
|
|||
# limitations under the License.
|
||||
# ============================================================================
|
||||
"""export checkpoint file into models"""
|
||||
|
||||
import os
|
||||
import numpy as np
|
||||
import mindspore.nn as nn
|
||||
from mindspore.common.tensor import Tensor
|
||||
|
|
@ -23,6 +23,7 @@ from mindspore.train.serialization import load_checkpoint, export, load_param_in
|
|||
from src.fasttext_model import FastText
|
||||
|
||||
from model_utils.config import config
|
||||
from model_utils.moxing_adapter import moxing_wrapper
|
||||
|
||||
if config.data_name == "ag":
|
||||
target_label1 = ['0', '1', '2', '3']
|
||||
|
|
@ -31,8 +32,6 @@ elif config.data_name == 'dbpedia':
|
|||
elif config.data_name == 'yelp_p':
|
||||
target_label1 = ['0', '1']
|
||||
|
||||
context.set_context(mode=context.GRAPH_MODE, save_graphs=False, device_target=config.device_target)
|
||||
|
||||
class FastTextInferExportCell(nn.Cell):
|
||||
"""
|
||||
Encapsulation class of FastText network infer.
|
||||
|
|
@ -81,5 +80,18 @@ def run_fasttext_export():
|
|||
src_tokens_length = Tensor(np.ones((src_tokens_length_shape)).astype(np.int32))
|
||||
export(ft_infer, src_tokens, src_tokens_length, file_name=file_name, file_format=config.file_format)
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
def modelarts_pre_process():
|
||||
'''modelarts pre process function.'''
|
||||
config.file_name = os.path.join(config.output_path, config.file_name)
|
||||
|
||||
|
||||
@moxing_wrapper(pre_process=modelarts_pre_process)
|
||||
def run_export():
|
||||
'''run export.'''
|
||||
context.set_context(mode=context.GRAPH_MODE, save_graphs=False, device_target=config.device_target)
|
||||
|
||||
run_fasttext_export()
|
||||
|
||||
if __name__ == '__main__':
|
||||
run_export()
|
||||
|
|
|
|||
|
|
@ -129,6 +129,22 @@ If you want to run in modelarts, please check the official documentation of [mod
|
|||
# (4) Set the startup file to "eval.py" on the website UI interface.
|
||||
# (5) Set the "Dataset path" and "Output file path" and "Job log path" to your path on the website UI interface.
|
||||
# (6) Create your job.
|
||||
|
||||
# run export on modelarts example
|
||||
# (1) Copy or upload your trained model to S3 bucket.
|
||||
# (2) Perform a or b.
|
||||
# a. Set "file_name='ncf'" on default_config.yaml file.
|
||||
# Set "file_format='MINDIR'" on default_config.yaml file.
|
||||
# Set "ckpt_file='/cache/checkpoint_path/model.ckpt'" on default_config.yaml file.
|
||||
# Set "checkpoint_url=/The path of checkpoint in S3/" on default_config.yaml file.
|
||||
# b. Add "file_name='ncf'" on the website UI interface.
|
||||
# Add "file_format='MINDIR'" on the website UI interface.
|
||||
# Add "ckpt_file='/cache/checkpoint_path/model.ckpt'" on the website UI interface.
|
||||
# Set "checkpoint_url=/The path of checkpoint in S3/" on the website UI interface.
|
||||
# (3) Set the code directory to "/path/ncf" on the website UI interface.
|
||||
# (4) Set the startup file to "export.py" on the website UI interface.
|
||||
# (5) Set the "Dataset path" and "Output file path" and "Job log path" to your path on the website UI interface.
|
||||
# (6) Create your job.
|
||||
```
|
||||
|
||||
# [Script Description](#contents)
|
||||
|
|
|
|||
|
|
@ -13,19 +13,29 @@
|
|||
# limitations under the License.
|
||||
# ============================================================================
|
||||
"""ncf export file"""
|
||||
import os
|
||||
import numpy as np
|
||||
|
||||
from mindspore import Tensor, context, load_checkpoint, load_param_into_net, export
|
||||
|
||||
import src.constants as rconst
|
||||
from model_utils.config import config
|
||||
from model_utils.moxing_adapter import moxing_wrapper
|
||||
from ncf import NCFModel, PredictWithSigmoid
|
||||
|
||||
context.set_context(mode=context.GRAPH_MODE, device_target=config.device_target)
|
||||
if config.device_target == "Ascend":
|
||||
context.set_context(device_id=config.device_id)
|
||||
|
||||
if __name__ == "__main__":
|
||||
def modelarts_pre_process():
|
||||
'''modelarts pre process function.'''
|
||||
config.file_name = os.path.join(config.output_path, config.file_name)
|
||||
|
||||
|
||||
@moxing_wrapper(pre_process=modelarts_pre_process)
|
||||
def run_export():
|
||||
'''run export.'''
|
||||
context.set_context(mode=context.GRAPH_MODE, device_target=config.device_target)
|
||||
if config.device_target == "Ascend":
|
||||
context.set_context(device_id=config.device_id)
|
||||
|
||||
topk = rconst.TOP_K
|
||||
num_eval_neg = rconst.NUM_EVAL_NEGATIVES
|
||||
|
||||
|
|
@ -57,3 +67,7 @@ if __name__ == "__main__":
|
|||
|
||||
input_data = [users, items, masks]
|
||||
export(network, *input_data, file_name=config.file_name, file_format=config.file_format)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run_export()
|
||||
|
|
|
|||
|
|
@ -185,7 +185,7 @@ The entire code structure is as following:
|
|||
|
||||
- Train on [ModelArts](https://support.huaweicloud.com/modelarts/)
|
||||
|
||||
```python
|
||||
```text
|
||||
# Train 8p with Ascend
|
||||
# (1) Perform a or b.
|
||||
# a. Set "enable_modelarts=True" on base_config.yaml file.
|
||||
|
|
@ -246,6 +246,29 @@ The entire code structure is as following:
|
|||
# (6) Set the startup file to "eval.py" on the website UI interface.
|
||||
# (7) Set the "Dataset path" and "Output file path" and "Job log path" to your path on the website UI interface.
|
||||
# (8) Create your job.
|
||||
#
|
||||
# Export 1p with Ascend
|
||||
# (1) Perform a or b.
|
||||
# a. Set "enable_modelarts=True" on base_config.yaml file.
|
||||
# Set "checkpoint_url='s3://dir_to_your_pretrain/'" on default_config.yaml file.
|
||||
# Set "pretrained='/cache/checkpoint_path/model.ckpt'" on default_config.yaml file.
|
||||
# Set "batch_size=1" on default_config.yaml file.
|
||||
# Set "file_format='AIR'" on default_config.yaml file.
|
||||
# Set "file_name='FaceDetection'" on default_config.yaml file.
|
||||
# Set other parameters on default_config.yaml file you need.
|
||||
# b. Add "enable_modelarts=True" on the website UI interface.
|
||||
# Add "checkpoint_url=s3://dir_to_your_pretrain/" on the website UI interface.
|
||||
# Add "pretrained=/cache/checkpoint_path/model.ckpt" on the website UI interface.
|
||||
# Add "batch_size=1" on the website UI interface.
|
||||
# Add "file_format=AIR" on the website UI interface.
|
||||
# Add "file_name=FaceDetection" on the website UI interface.
|
||||
# Add other parameters on the website UI interface.
|
||||
# (3) Upload or copy your trained model to S3 bucket.
|
||||
# (4) Upload a zip dataset to S3 bucket. (you could also upload the origin dataset, but it can be so slow.)
|
||||
# (5) Set the code directory to "/path/FaceDetection" on the website UI interface.
|
||||
# (6) Set the startup file to "export.py" on the website UI interface.
|
||||
# (7) Set the "Dataset path" and "Output file path" and "Job log path" to your path on the website UI interface.
|
||||
# (8) Create your job.
|
||||
```
|
||||
|
||||
### Evaluation
|
||||
|
|
|
|||
|
|
@ -22,8 +22,16 @@ from mindspore.train.serialization import export, load_checkpoint, load_param_in
|
|||
from src.network_define import BuildTestNetwork
|
||||
from src.FaceDetection.yolov3 import HwYolov3 as backbone_HwYolov3
|
||||
from model_utils.config import config
|
||||
from model_utils.moxing_adapter import moxing_wrapper
|
||||
|
||||
def save_air():
|
||||
|
||||
def modelarts_pre_process():
|
||||
'''modelarts pre process function.'''
|
||||
config.file_name = os.path.join(config.output_path, config.file_name)
|
||||
|
||||
|
||||
@moxing_wrapper(pre_process=modelarts_pre_process)
|
||||
def run_export():
|
||||
'''save air or mindir'''
|
||||
anchors = config.anchors
|
||||
reduction_0 = 64.0
|
||||
|
|
@ -62,4 +70,4 @@ def save_air():
|
|||
|
||||
|
||||
if __name__ == "__main__":
|
||||
save_air()
|
||||
run_export()
|
||||
|
|
|
|||
|
|
@ -312,6 +312,23 @@ cd ./scripts
|
|||
sh run_export.sh 1 0 ./0-1_1.ckpt
|
||||
```
|
||||
|
||||
```python
|
||||
# run export on modelarts example
|
||||
# (1) Copy or upload your trained model to S3 bucket.
|
||||
# (2) Add "config_path='/path_to_code/inference_config.yaml'" on the website UI interface.
|
||||
# (3) Perform a or b.
|
||||
# a. Set "pretrained='/cache/checkpoint_path/model.ckpt'" on inference_config.yaml file.
|
||||
# Set "checkpoint_url='/The path of checkpoint in S3/'" on inference_config.yaml file.
|
||||
# Set "batch_size=1" on inference_config.yaml file.
|
||||
# b. Add "pretrained=/cache/checkpoint_path/model.ckpt" on the website UI interface.
|
||||
# Add "checkpoint_url=/The path of checkpoint in S3/" on the website UI interface.
|
||||
# Add "batch_size=1" on the website UI interface.
|
||||
# (4) Set the code directory to "/path/FaceRecognition" on the website UI interface.
|
||||
# (5) Set the startup file to "export.py" on the website UI interface.
|
||||
# (6) Set the "Dataset path" and "Output file path" and "Job log path" to your path on the website UI interface.
|
||||
# (7) Create your job.
|
||||
```
|
||||
|
||||
### Inference
|
||||
|
||||
```bash
|
||||
|
|
|
|||
|
|
@ -61,6 +61,19 @@ max_ckpts: -1
|
|||
dynamic_init_loss_scale: 65536
|
||||
ckpt_steps: 1000
|
||||
|
||||
# export option
|
||||
batch_size: 16
|
||||
file_name: "fr.midnir"
|
||||
file_format: "MINDIR"
|
||||
export_pre_bn: 0
|
||||
export_inference: 1
|
||||
export_use_se: 0
|
||||
export_emb_size: 256
|
||||
export_act_type: "relu"
|
||||
export_backbone: "r100"
|
||||
export_use_drop: 0
|
||||
head: "0"
|
||||
|
||||
---
|
||||
|
||||
# Help description for each configuration
|
||||
|
|
@ -73,4 +86,17 @@ device_target: 'Target device type'
|
|||
enable_profiling: 'Whether enable profiling while training, default: False'
|
||||
|
||||
train_stage: "Train stage, base or beta"
|
||||
is_distributed: "If multi device"
|
||||
is_distributed: "If multi device"
|
||||
|
||||
# export option
|
||||
batch_size: "batch size"
|
||||
file_name: "file name"
|
||||
file_format: "file format, choices in ['MINDIR', 'AIR']"
|
||||
export_pre_bn: "1: bn-conv-bn-conv-bn, 0: conv-bn-conv-bn"
|
||||
export_inference: "use inference backbone"
|
||||
export_use_se: "use se block or not"
|
||||
export_emb_size: "embedding size of the network"
|
||||
export_act_type: "activation layer type"
|
||||
export_backbone: "backbone network"
|
||||
export_use_drop: "whether use dropout in network"
|
||||
head: "head type, default is 0"
|
||||
|
|
@ -61,6 +61,19 @@ max_ckpts: -1
|
|||
dynamic_init_loss_scale: 65536
|
||||
ckpt_steps: 1000
|
||||
|
||||
# export option
|
||||
batch_size: 16
|
||||
file_name: "fr.midnir"
|
||||
file_format: "MINDIR"
|
||||
export_pre_bn: 0
|
||||
export_inference: 1
|
||||
export_use_se: 0
|
||||
export_emb_size: 256
|
||||
export_act_type: "relu"
|
||||
export_backbone: "r100"
|
||||
export_use_drop: 0
|
||||
head: "0"
|
||||
|
||||
---
|
||||
|
||||
# Help description for each configuration
|
||||
|
|
@ -73,4 +86,17 @@ device_target: 'Target device type'
|
|||
enable_profiling: 'Whether enable profiling while training, default: False'
|
||||
|
||||
train_stage: "Train stage, base or beta"
|
||||
is_distributed: "If multi device"
|
||||
is_distributed: "If multi device"
|
||||
|
||||
# export option
|
||||
batch_size: "batch size"
|
||||
file_name: "file name"
|
||||
file_format: "file format, choices in ['MINDIR', 'AIR']"
|
||||
export_pre_bn: "1: bn-conv-bn-conv-bn, 0: conv-bn-conv-bn"
|
||||
export_inference: "use inference backbone"
|
||||
export_use_se: "use se block or not"
|
||||
export_emb_size: "embedding size of the network"
|
||||
export_act_type: "activation layer type"
|
||||
export_backbone: "backbone network"
|
||||
export_use_drop: "whether use dropout in network"
|
||||
head: "head type, default is 0"
|
||||
|
|
@ -61,6 +61,19 @@ max_ckpts: -1
|
|||
dynamic_init_loss_scale: 65536
|
||||
ckpt_steps: 1000
|
||||
|
||||
# export option
|
||||
batch_size: 16
|
||||
file_name: "fr.midnir"
|
||||
file_format: "MINDIR"
|
||||
export_pre_bn: 0
|
||||
export_inference: 1
|
||||
export_use_se: 0
|
||||
export_emb_size: 256
|
||||
export_act_type: "relu"
|
||||
export_backbone: "r100"
|
||||
export_use_drop: 0
|
||||
head: "0"
|
||||
|
||||
---
|
||||
|
||||
# Help description for each configuration
|
||||
|
|
@ -73,4 +86,17 @@ device_target: 'Target device type'
|
|||
enable_profiling: 'Whether enable profiling while training, default: False'
|
||||
|
||||
train_stage: "Train stage, base or beta"
|
||||
is_distributed: "If multi device"
|
||||
is_distributed: "If multi device"
|
||||
|
||||
# export option
|
||||
batch_size: "batch size"
|
||||
file_name: "file name"
|
||||
file_format: "file format, choices in ['MINDIR', 'AIR']"
|
||||
export_pre_bn: "1: bn-conv-bn-conv-bn, 0: conv-bn-conv-bn"
|
||||
export_inference: "use inference backbone"
|
||||
export_use_se: "use se block or not"
|
||||
export_emb_size: "embedding size of the network"
|
||||
export_act_type: "activation layer type"
|
||||
export_backbone: "backbone network"
|
||||
export_use_drop: "whether use dropout in network"
|
||||
head: "head type, default is 0"
|
||||
|
|
@ -61,6 +61,19 @@ max_ckpts: -1
|
|||
dynamic_init_loss_scale: 65536
|
||||
ckpt_steps: 1000
|
||||
|
||||
# export option
|
||||
batch_size: 16
|
||||
file_name: "fr.midnir"
|
||||
file_format: "MINDIR"
|
||||
export_pre_bn: 0
|
||||
export_inference: 1
|
||||
export_use_se: 0
|
||||
export_emb_size: 256
|
||||
export_act_type: "relu"
|
||||
export_backbone: "r100"
|
||||
export_use_drop: 0
|
||||
head: "0"
|
||||
|
||||
---
|
||||
|
||||
# Help description for each configuration
|
||||
|
|
@ -73,4 +86,17 @@ device_target: 'Target device type'
|
|||
enable_profiling: 'Whether enable profiling while training, default: False'
|
||||
|
||||
train_stage: "Train stage, base or beta"
|
||||
is_distributed: "If multi device"
|
||||
is_distributed: "If multi device"
|
||||
|
||||
# export option
|
||||
batch_size: "batch size"
|
||||
file_name: "file name"
|
||||
file_format: "file format, choices in ['MINDIR', 'AIR']"
|
||||
export_pre_bn: "1: bn-conv-bn-conv-bn, 0: conv-bn-conv-bn"
|
||||
export_inference: "use inference backbone"
|
||||
export_use_se: "use se block or not"
|
||||
export_emb_size: "embedding size of the network"
|
||||
export_act_type: "activation layer type"
|
||||
export_backbone: "backbone network"
|
||||
export_use_drop: "whether use dropout in network"
|
||||
head: "head type, default is 0"
|
||||
|
|
@ -14,7 +14,6 @@
|
|||
# ============================================================================
|
||||
"""Convert ckpt to air."""
|
||||
import os
|
||||
import argparse
|
||||
import numpy as np
|
||||
|
||||
from mindspore import context
|
||||
|
|
@ -22,15 +21,32 @@ from mindspore import Tensor
|
|||
from mindspore.train.serialization import export, load_checkpoint, load_param_into_net
|
||||
|
||||
from src.backbone.resnet import get_backbone
|
||||
|
||||
devid = 0
|
||||
context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", save_graphs=False, device_id=devid)
|
||||
from model_utils.config import config
|
||||
from model_utils.moxing_adapter import moxing_wrapper
|
||||
|
||||
|
||||
def main(args):
|
||||
network = get_backbone(args)
|
||||
def modelarts_pre_process():
|
||||
'''modelarts pre process function.'''
|
||||
config.file_name = os.path.join(config.output_path, config.file_name)
|
||||
|
||||
ckpt_path = args.pretrained
|
||||
|
||||
@moxing_wrapper(pre_process=modelarts_pre_process)
|
||||
def run_export():
|
||||
'''run export.'''
|
||||
config.pre_bn = config.export_pre_bn
|
||||
config.inference = config.export_inference
|
||||
config.use_se = config.export_use_se
|
||||
config.emb_size = config.export_emb_size
|
||||
config.act_type = config.export_act_type
|
||||
config.backbone = config.export_backbone
|
||||
config.use_drop = config.export_use_drop
|
||||
|
||||
devid = 0
|
||||
context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", save_graphs=False, device_id=devid)
|
||||
|
||||
network = get_backbone(config)
|
||||
|
||||
ckpt_path = config.pretrained
|
||||
if os.path.isfile(ckpt_path):
|
||||
param_dict = load_checkpoint(ckpt_path)
|
||||
param_dict_new = {}
|
||||
|
|
@ -49,35 +65,14 @@ def main(args):
|
|||
network.add_flags_recursive(fp16=True)
|
||||
network.set_train(False)
|
||||
|
||||
input_data = np.random.uniform(low=0, high=1.0, size=(args.batch_size, 3, 112, 112)).astype(np.float32)
|
||||
input_data = np.random.uniform(low=0, high=1.0, size=(config.batch_size, 3, 112, 112)).astype(np.float32)
|
||||
tensor_input_data = Tensor(input_data)
|
||||
|
||||
file_path = ckpt_path
|
||||
export(network, tensor_input_data, file_name=args.file_name, file_format=args.file_format)
|
||||
export(network, tensor_input_data, file_name=config.file_name, file_format=config.file_format)
|
||||
print('-----------------------export model success, save file:{}-----------------------'.format(file_path))
|
||||
|
||||
|
||||
def parse_args():
|
||||
'''parse_args'''
|
||||
parser = argparse.ArgumentParser(description='Convert ckpt to air')
|
||||
parser.add_argument('--pretrained', type=str, default='', help='pretrained model to load')
|
||||
parser.add_argument('--batch_size', type=int, default=16, help='batch size')
|
||||
parser.add_argument('--pre_bn', type=int, default=0, help='1: bn-conv-bn-conv-bn, 0: conv-bn-conv-bn')
|
||||
parser.add_argument('--inference', type=int, default=1, help='use inference backbone')
|
||||
parser.add_argument('--use_se', type=int, default=0, help='use se block or not')
|
||||
parser.add_argument('--emb_size', type=int, default=256, help='embedding size of the network')
|
||||
parser.add_argument('--act_type', type=str, default='relu', help='activation layer type')
|
||||
parser.add_argument('--backbone', type=str, default='r100', help='backbone network')
|
||||
parser.add_argument('--head', type=str, default='0', help='head type, default is 0')
|
||||
parser.add_argument('--use_drop', type=int, default=0, help='whether use dropout in network')
|
||||
parser.add_argument('--file_name', type=str, default='fr.midnir', help='file name')
|
||||
parser.add_argument('--file_format', type=str, default='MINDIR', choices=['MINDIR', 'AIR'], help='file format')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
return args
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
arg = parse_args()
|
||||
main(arg)
|
||||
run_export()
|
||||
|
|
|
|||
|
|
@ -48,6 +48,19 @@ test_img_list: ""
|
|||
dis_img_predix: ""
|
||||
dis_img_list: ""
|
||||
|
||||
# export option
|
||||
batch_size: 16
|
||||
file_name: "fr.midnir"
|
||||
file_format: "MINDIR"
|
||||
export_pre_bn: 0
|
||||
export_inference: 1
|
||||
export_use_se: 0
|
||||
export_emb_size: 256
|
||||
export_act_type: "relu"
|
||||
export_backbone: "r100"
|
||||
export_use_drop: 0
|
||||
head: "0"
|
||||
|
||||
---
|
||||
|
||||
# Help description for each configuration
|
||||
|
|
@ -57,4 +70,17 @@ train_url: "Url for modelarts"
|
|||
data_path: "The location of the input data."
|
||||
output_path: "The location of the output file."
|
||||
device_target: 'Target device type'
|
||||
enable_profiling: 'Whether enable profiling while training, default: False'
|
||||
enable_profiling: 'Whether enable profiling while training, default: False'
|
||||
|
||||
# export option
|
||||
batch_size: "batch size"
|
||||
file_name: "file name"
|
||||
file_format: "file format, choices in ['MINDIR', 'AIR']"
|
||||
export_pre_bn: "1: bn-conv-bn-conv-bn, 0: conv-bn-conv-bn"
|
||||
export_inference: "use inference backbone"
|
||||
export_use_se: "use se block or not"
|
||||
export_emb_size: "embedding size of the network"
|
||||
export_act_type: "activation layer type"
|
||||
export_backbone: "backbone network"
|
||||
export_use_drop: "whether use dropout in network"
|
||||
head: "head type, default is 0"
|
||||
|
|
@ -48,6 +48,19 @@ test_img_list: ""
|
|||
dis_img_predix: ""
|
||||
dis_img_list: ""
|
||||
|
||||
# export option
|
||||
batch_size: 16
|
||||
file_name: "fr.midnir"
|
||||
file_format: "MINDIR"
|
||||
export_pre_bn: 0
|
||||
export_inference: 1
|
||||
export_use_se: 0
|
||||
export_emb_size: 256
|
||||
export_act_type: "relu"
|
||||
export_backbone: "r100"
|
||||
export_use_drop: 0
|
||||
head: "0"
|
||||
|
||||
---
|
||||
|
||||
# Help description for each configuration
|
||||
|
|
@ -57,4 +70,17 @@ train_url: "Url for modelarts"
|
|||
data_path: "The location of the input data."
|
||||
output_path: "The location of the output file."
|
||||
device_target: 'Target device type'
|
||||
enable_profiling: 'Whether enable profiling while training, default: False'
|
||||
enable_profiling: 'Whether enable profiling while training, default: False'
|
||||
|
||||
# export option
|
||||
batch_size: "batch size"
|
||||
file_name: "file name"
|
||||
file_format: "file format, choices in ['MINDIR', 'AIR']"
|
||||
export_pre_bn: "1: bn-conv-bn-conv-bn, 0: conv-bn-conv-bn"
|
||||
export_inference: "use inference backbone"
|
||||
export_use_se: "use se block or not"
|
||||
export_emb_size: "embedding size of the network"
|
||||
export_act_type: "activation layer type"
|
||||
export_backbone: "backbone network"
|
||||
export_use_drop: "whether use dropout in network"
|
||||
head: "head type, default is 0"
|
||||
Loading…
Reference in New Issue