p60975123
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

!11950 Overwrite default port/hostname from env variables for cache_admin 

From: @lixiachen
Reviewed-by: 
Signed-off-by:
This commit is contained in:
mindspore-ci-bot 2021-02-03 23:38:48 +08:00 committed by Gitee
parent 62d272f648 97cd142f8d
commit e8f4cafd02
13 changed files with 126 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
mindspore/ccsrc/minddata/dataset/core/config_manager.cc
View File

@ -1,5 +1,5 @@
/**
* Copyright 2019 Huawei Technologies Co., Ltd
* Copyright 2019-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -57,9 +57,8 @@ ConfigManager::ConfigManager()
char *end = nullptr;
cache_port_ = strtol(env_cache_port, &end, 10);
if (*end != '\0') {
MS_LOG(WARNING) << "\nCache port from env variable MS_CACHE_PORT is invalid, back to use default "
<< kCfgDefaultCachePort << std::endl;
cache_port_ = kCfgDefaultCachePort;
MS_LOG(WARNING) << "Cache port from env variable MS_CACHE_PORT is invalid\n";
cache_port_ = 0; // cause the port range validation to generate an error during the validation checks
}
}
}

15
mindspore/ccsrc/minddata/dataset/engine/cache/cache_admin_arg.cc vendored
View File

@ -1,5 +1,5 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -50,6 +50,19 @@ CacheAdminArgHandler::CacheAdminArgHandler()
hostname_(kCfgDefaultCacheHost),
spill_dir_(""),
command_id_(CommandId::kCmdUnknown) {
const char *env_cache_host = std::getenv("MS_CACHE_HOST");
const char *env_cache_port = std::getenv("MS_CACHE_PORT");
if (env_cache_host != nullptr) {
hostname_ = env_cache_host;
}
if (env_cache_port != nullptr) {
char *end = nullptr;
port_ = strtol(env_cache_port, &end, 10);
if (*end != '\0') {
std::cerr << "Cache port from env variable MS_CACHE_PORT is invalid\n";
port_ = 0; // cause the port range validation to generate an error during the validation checks
}
}
// Initialize the command mappings
arg_map_["-h"] = ArgValue::kArgHost;
arg_map_["--hostname"] = ArgValue::kArgHost;

2
mindspore/ccsrc/minddata/dataset/engine/cache/cache_admin_arg.h vendored
View File

@ -1,5 +1,5 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

2
mindspore/ccsrc/minddata/dataset/engine/cache/cache_main.cc vendored
View File

@ -1,5 +1,5 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

2
mindspore/ccsrc/minddata/dataset/engine/cache/cache_request.cc vendored
View File

@ -1,5 +1,5 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

2
mindspore/ccsrc/minddata/dataset/engine/cache/cache_request.h vendored
View File

@ -1,5 +1,5 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

2
mindspore/ccsrc/minddata/dataset/engine/cache/cache_server.cc vendored
View File

@ -1,5 +1,5 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

2
mindspore/ccsrc/minddata/dataset/engine/cache/cache_server.h vendored
View File

@ -1,5 +1,5 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

2
mindspore/ccsrc/minddata/dataset/engine/cache/de_tensor.fbs vendored
View File

@ -1,5 +1,5 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

4
mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_base_op.cc
View File

@ -1,5 +1,5 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -307,7 +307,7 @@ Status CacheBase::Prefetcher(int32_t worker_id) {
if (rc == StatusCode::kMDNetWorkError && retry_count < max_retries) {
// If we get some network error, we will attempt some retries
retry_count++;
} else if (rc.IsError()) {
} else if (rc.IsError() && rc.StatusCode() != StatusCode::kMDInterrupted) {
MS_LOG(WARNING) << rc.ToString();
return rc;
}

10
mindspore/dataset/engine/cache_client.py
View File

@ -1,4 +1,4 @@
# Copyright 2019 Huawei Technologies Co., Ltd
# Copyright 2019-2021 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -38,6 +38,14 @@ class DatasetCache:
num_connections (int, optional): Number of tcp/ip connections (default=12).
prefetch_size (int, optional): Prefetch size (default=20).
Examples:
>>> import mindspore.dataset as ds
>>>
>>> # create a cache instance, in which session_id is generated from command line `cache_admin -g`
>>> some_cache = ds.DatasetCache(session_id=session_id, size=0)
>>>
>>> dataset_dir = "path/to/imagefolder_directory"
>>> ds1 = ds.ImageFolderDataset(dataset_dir, cache=some_cache)
"""
def __init__(self, session_id, size=0, spilling=False, hostname=None, port=None, num_connections=None,

48
tests/ut/python/dataset/test_cache_map.py
View File

@ -2110,11 +2110,59 @@ def test_cache_map_nested_repeat():
if __name__ == '__main__':
# This is just a list of tests, don't try to run these tests with 'python test_cache_map.py'
# since cache server is required to be brought up first
test_cache_map_basic1()
test_cache_map_basic2()
test_cache_map_basic3()
test_cache_map_basic4()
test_cache_map_basic5()
test_cache_map_failure1()
test_cache_map_failure2()
test_cache_map_failure3()
test_cache_map_failure4()
test_cache_map_failure5()
test_cache_map_failure6()
test_cache_map_failure7()
test_cache_map_failure8()
test_cache_map_failure9()
test_cache_map_failure10()
test_cache_map_failure11()
test_cache_map_split1()
test_cache_map_split2()
test_cache_map_parameter_check()
test_cache_map_running_twice1()
test_cache_map_running_twice2()
test_cache_map_extra_small_size1()
test_cache_map_extra_small_size2()
test_cache_map_no_image()
test_cache_map_parallel_pipeline1(shard=0)
test_cache_map_parallel_pipeline2(shard=1)
test_cache_map_parallel_workers()
test_cache_map_server_workers_1()
test_cache_map_server_workers_100()
test_cache_map_num_connections_1()
test_cache_map_num_connections_100()
test_cache_map_prefetch_size_1()
test_cache_map_prefetch_size_100()
test_cache_map_to_device()
test_cache_map_epoch_ctrl1()
test_cache_map_epoch_ctrl2()
test_cache_map_epoch_ctrl3()
test_cache_map_coco1()
test_cache_map_coco2()
test_cache_map_mnist1()
test_cache_map_mnist2()
test_cache_map_celeba1()
test_cache_map_celeba2()
test_cache_map_manifest1()
test_cache_map_manifest2()
test_cache_map_cifar1()
test_cache_map_cifar2()
test_cache_map_cifar3()
test_cache_map_cifar4()
test_cache_map_voc1()
test_cache_map_voc2()
test_cache_map_python_sampler1()
test_cache_map_python_sampler2()
test_cache_map_nested_repeat()

43
tests/ut/python/dataset/test_cache_nomap.py
View File

@ -2125,6 +2125,8 @@ def test_cache_nomap_failure5():
if __name__ == '__main__':
# This is just a list of tests, don't try to run these tests with 'python test_cache_nomap.py'
# since cache server is required to be brought up first
test_cache_nomap_basic1()
test_cache_nomap_basic2()
test_cache_nomap_basic3()
@ -2132,8 +2134,49 @@ if __name__ == '__main__':
test_cache_nomap_basic5()
test_cache_nomap_basic6()
test_cache_nomap_basic7()
test_cache_nomap_basic8()
test_cache_nomap_basic9()
test_cache_nomap_allowed_share1()
test_cache_nomap_allowed_share2()
test_cache_nomap_allowed_share3()
test_cache_nomap_allowed_share4()
test_cache_nomap_disallowed_share1()
test_cache_nomap_running_twice1()
test_cache_nomap_running_twice2()
test_cache_nomap_extra_small_size1()
test_cache_nomap_extra_small_size2()
test_cache_nomap_parallel_pipeline1(shard=0)
test_cache_nomap_parallel_pipeline2(shard=1)
test_cache_nomap_parallel_workers()
test_cache_nomap_server_workers_1()
test_cache_nomap_server_workers_100()
test_cache_nomap_num_connections_1()
test_cache_nomap_num_connections_100()
test_cache_nomap_prefetch_size_1()
test_cache_nomap_prefetch_size_100()
test_cache_nomap_to_device()
test_cache_nomap_session_destroy()
test_cache_nomap_server_stop()
test_cache_nomap_epoch_ctrl1()
test_cache_nomap_epoch_ctrl2()
test_cache_nomap_epoch_ctrl3()
test_cache_nomap_epoch_ctrl4()
test_cache_nomap_multiple_cache1()
test_cache_nomap_multiple_cache2()
test_cache_nomap_multiple_cache3()
test_cache_nomap_multiple_cache_train()
test_cache_nomap_multiple_cache_eval()
test_cache_nomap_clue1()
test_cache_nomap_clue2()
test_cache_nomap_csv1()
test_cache_nomap_csv2()
test_cache_nomap_textfile1()
test_cache_nomap_textfile2()
test_cache_nomap_nested_repeat()
test_cache_nomap_get_repeat_count()
test_cache_nomap_long_file_list()
test_cache_nomap_failure1()
test_cache_nomap_failure2()
test_cache_nomap_failure3()
test_cache_nomap_failure4()
test_cache_nomap_failure5()