diff --git a/mindspore/ccsrc/minddata/dataset/core/config_manager.cc b/mindspore/ccsrc/minddata/dataset/core/config_manager.cc index 404ba49cd01..995001328b4 100644 --- a/mindspore/ccsrc/minddata/dataset/core/config_manager.cc +++ b/mindspore/ccsrc/minddata/dataset/core/config_manager.cc @@ -1,5 +1,5 @@ /** - * Copyright 2019 Huawei Technologies Co., Ltd + * Copyright 2019-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -57,9 +57,8 @@ ConfigManager::ConfigManager() char *end = nullptr; cache_port_ = strtol(env_cache_port, &end, 10); if (*end != '\0') { - MS_LOG(WARNING) << "\nCache port from env variable MS_CACHE_PORT is invalid, back to use default " - << kCfgDefaultCachePort << std::endl; - cache_port_ = kCfgDefaultCachePort; + MS_LOG(WARNING) << "Cache port from env variable MS_CACHE_PORT is invalid\n"; + cache_port_ = 0; // cause the port range validation to generate an error during the validation checks } } } diff --git a/mindspore/ccsrc/minddata/dataset/engine/cache/cache_admin_arg.cc b/mindspore/ccsrc/minddata/dataset/engine/cache/cache_admin_arg.cc index 5774c80e410..ca20b29a373 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/cache/cache_admin_arg.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/cache/cache_admin_arg.cc @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2020-2021 Huawei Technologies Co., Ltd * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -50,6 +50,19 @@ CacheAdminArgHandler::CacheAdminArgHandler() hostname_(kCfgDefaultCacheHost), spill_dir_(""), command_id_(CommandId::kCmdUnknown) { + const char *env_cache_host = std::getenv("MS_CACHE_HOST"); + const char *env_cache_port = std::getenv("MS_CACHE_PORT"); + if (env_cache_host != nullptr) { + hostname_ = env_cache_host; + } + if (env_cache_port != nullptr) { + char *end = nullptr; + port_ = strtol(env_cache_port, &end, 10); + if (*end != '\0') { + std::cerr << "Cache port from env variable MS_CACHE_PORT is invalid\n"; + port_ = 0; // cause the port range validation to generate an error during the validation checks + } + } // Initialize the command mappings arg_map_["-h"] = ArgValue::kArgHost; arg_map_["--hostname"] = ArgValue::kArgHost; diff --git a/mindspore/ccsrc/minddata/dataset/engine/cache/cache_admin_arg.h b/mindspore/ccsrc/minddata/dataset/engine/cache/cache_admin_arg.h index 6b95817f3db..b9ed50d12cc 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/cache/cache_admin_arg.h +++ b/mindspore/ccsrc/minddata/dataset/engine/cache/cache_admin_arg.h @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2020-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/mindspore/ccsrc/minddata/dataset/engine/cache/cache_main.cc b/mindspore/ccsrc/minddata/dataset/engine/cache/cache_main.cc index a118a5e6091..172362c6e35 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/cache/cache_main.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/cache/cache_main.cc @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2020-2021 Huawei Technologies Co., Ltd * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/mindspore/ccsrc/minddata/dataset/engine/cache/cache_request.cc b/mindspore/ccsrc/minddata/dataset/engine/cache/cache_request.cc index 14803bcf28a..d0a664a4778 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/cache/cache_request.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/cache/cache_request.cc @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2020-2021 Huawei Technologies Co., Ltd * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/mindspore/ccsrc/minddata/dataset/engine/cache/cache_request.h b/mindspore/ccsrc/minddata/dataset/engine/cache/cache_request.h index 36bdc5abac5..fa961dff184 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/cache/cache_request.h +++ b/mindspore/ccsrc/minddata/dataset/engine/cache/cache_request.h @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2020-2021 Huawei Technologies Co., Ltd * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/mindspore/ccsrc/minddata/dataset/engine/cache/cache_server.cc b/mindspore/ccsrc/minddata/dataset/engine/cache/cache_server.cc index 14c38302092..e4608dd06ac 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/cache/cache_server.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/cache/cache_server.cc @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2020-2021 Huawei Technologies Co., Ltd * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/mindspore/ccsrc/minddata/dataset/engine/cache/cache_server.h b/mindspore/ccsrc/minddata/dataset/engine/cache/cache_server.h index c8f2b95a980..156f4b242e2 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/cache/cache_server.h +++ b/mindspore/ccsrc/minddata/dataset/engine/cache/cache_server.h @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2020-2021 Huawei Technologies Co., Ltd * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/mindspore/ccsrc/minddata/dataset/engine/cache/de_tensor.fbs b/mindspore/ccsrc/minddata/dataset/engine/cache/de_tensor.fbs index 59b76003b95..febb3d43acb 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/cache/de_tensor.fbs +++ b/mindspore/ccsrc/minddata/dataset/engine/cache/de_tensor.fbs @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2020-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_base_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_base_op.cc index 5accad7c76e..156e16341bb 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_base_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_base_op.cc @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2020-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -307,7 +307,7 @@ Status CacheBase::Prefetcher(int32_t worker_id) { if (rc == StatusCode::kMDNetWorkError && retry_count < max_retries) { // If we get some network error, we will attempt some retries retry_count++; - } else if (rc.IsError()) { + } else if (rc.IsError() && rc.StatusCode() != StatusCode::kMDInterrupted) { MS_LOG(WARNING) << rc.ToString(); return rc; } diff --git a/mindspore/dataset/engine/cache_client.py b/mindspore/dataset/engine/cache_client.py index 84edecc0ed1..f9ab70527a6 100644 --- a/mindspore/dataset/engine/cache_client.py +++ b/mindspore/dataset/engine/cache_client.py @@ -1,4 +1,4 @@ -# Copyright 2019 Huawei Technologies Co., Ltd +# Copyright 2019-2021 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -38,6 +38,14 @@ class DatasetCache: num_connections (int, optional): Number of tcp/ip connections (default=12). prefetch_size (int, optional): Prefetch size (default=20). + Examples: + >>> import mindspore.dataset as ds + >>> + >>> # create a cache instance, in which session_id is generated from command line `cache_admin -g` + >>> some_cache = ds.DatasetCache(session_id=session_id, size=0) + >>> + >>> dataset_dir = "path/to/imagefolder_directory" + >>> ds1 = ds.ImageFolderDataset(dataset_dir, cache=some_cache) """ def __init__(self, session_id, size=0, spilling=False, hostname=None, port=None, num_connections=None, diff --git a/tests/ut/python/dataset/test_cache_map.py b/tests/ut/python/dataset/test_cache_map.py index dedef3a349a..c7f321de625 100644 --- a/tests/ut/python/dataset/test_cache_map.py +++ b/tests/ut/python/dataset/test_cache_map.py @@ -2110,11 +2110,59 @@ def test_cache_map_nested_repeat(): if __name__ == '__main__': + # This is just a list of tests, don't try to run these tests with 'python test_cache_map.py' + # since cache server is required to be brought up first test_cache_map_basic1() test_cache_map_basic2() test_cache_map_basic3() test_cache_map_basic4() + test_cache_map_basic5() test_cache_map_failure1() test_cache_map_failure2() test_cache_map_failure3() test_cache_map_failure4() + test_cache_map_failure5() + test_cache_map_failure6() + test_cache_map_failure7() + test_cache_map_failure8() + test_cache_map_failure9() + test_cache_map_failure10() + test_cache_map_failure11() + test_cache_map_split1() + test_cache_map_split2() + test_cache_map_parameter_check() + test_cache_map_running_twice1() + test_cache_map_running_twice2() + test_cache_map_extra_small_size1() + test_cache_map_extra_small_size2() + test_cache_map_no_image() + test_cache_map_parallel_pipeline1(shard=0) + test_cache_map_parallel_pipeline2(shard=1) + test_cache_map_parallel_workers() + test_cache_map_server_workers_1() + test_cache_map_server_workers_100() + test_cache_map_num_connections_1() + test_cache_map_num_connections_100() + test_cache_map_prefetch_size_1() + test_cache_map_prefetch_size_100() + test_cache_map_to_device() + test_cache_map_epoch_ctrl1() + test_cache_map_epoch_ctrl2() + test_cache_map_epoch_ctrl3() + test_cache_map_coco1() + test_cache_map_coco2() + test_cache_map_mnist1() + test_cache_map_mnist2() + test_cache_map_celeba1() + test_cache_map_celeba2() + test_cache_map_manifest1() + test_cache_map_manifest2() + test_cache_map_cifar1() + test_cache_map_cifar2() + test_cache_map_cifar3() + test_cache_map_cifar4() + test_cache_map_voc1() + test_cache_map_voc2() + test_cache_map_python_sampler1() + test_cache_map_python_sampler2() + test_cache_map_nested_repeat() diff --git a/tests/ut/python/dataset/test_cache_nomap.py b/tests/ut/python/dataset/test_cache_nomap.py index 915e14bdd80..d4c405afe74 100644 --- a/tests/ut/python/dataset/test_cache_nomap.py +++ b/tests/ut/python/dataset/test_cache_nomap.py @@ -2125,6 +2125,8 @@ def test_cache_nomap_failure5(): if __name__ == '__main__': + # This is just a list of tests, don't try to run these tests with 'python test_cache_nomap.py' + # since cache server is required to be brought up first test_cache_nomap_basic1() test_cache_nomap_basic2() test_cache_nomap_basic3() @@ -2132,8 +2134,49 @@ if __name__ == '__main__': test_cache_nomap_basic5() test_cache_nomap_basic6() test_cache_nomap_basic7() + test_cache_nomap_basic8() + test_cache_nomap_basic9() test_cache_nomap_allowed_share1() test_cache_nomap_allowed_share2() test_cache_nomap_allowed_share3() test_cache_nomap_allowed_share4() test_cache_nomap_disallowed_share1() + test_cache_nomap_running_twice1() + test_cache_nomap_running_twice2() + test_cache_nomap_extra_small_size1() + test_cache_nomap_extra_small_size2() + test_cache_nomap_parallel_pipeline1(shard=0) + test_cache_nomap_parallel_pipeline2(shard=1) + test_cache_nomap_parallel_workers() + test_cache_nomap_server_workers_1() + test_cache_nomap_server_workers_100() + test_cache_nomap_num_connections_1() + test_cache_nomap_num_connections_100() + test_cache_nomap_prefetch_size_1() + test_cache_nomap_prefetch_size_100() + test_cache_nomap_to_device() + test_cache_nomap_session_destroy() + test_cache_nomap_server_stop() + test_cache_nomap_epoch_ctrl1() + test_cache_nomap_epoch_ctrl2() + test_cache_nomap_epoch_ctrl3() + test_cache_nomap_epoch_ctrl4() + test_cache_nomap_multiple_cache1() + test_cache_nomap_multiple_cache2() + test_cache_nomap_multiple_cache3() + test_cache_nomap_multiple_cache_train() + test_cache_nomap_multiple_cache_eval() + test_cache_nomap_clue1() + test_cache_nomap_clue2() + test_cache_nomap_csv1() + test_cache_nomap_csv2() + test_cache_nomap_textfile1() + test_cache_nomap_textfile2() + test_cache_nomap_nested_repeat() + test_cache_nomap_get_repeat_count() + test_cache_nomap_long_file_list() + test_cache_nomap_failure1() + test_cache_nomap_failure2() + test_cache_nomap_failure3() + test_cache_nomap_failure4() + test_cache_nomap_failure5()