Nigel
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

!30260 [MD][AutoTune] Re-enable AT for non-sink models 

Merge pull request !30260 from danishfarid/re-enable_AT
This commit is contained in:
i-robot 2022-02-25 22:26:14 +00:00 committed by Gitee
parent 09f114e52e 6c4697fc8b
commit 5deccfe64b
No known key found for this signature in database
GPG Key ID: 173E9B9CA92EEF8F
2 changed files with 38 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
mindspore/ccsrc/minddata/dataset/engine/perf/auto_tune.cc
View File

@ -118,12 +118,6 @@ Status AutoTune::CollectOpsInfo() {
// sort parallel ops in reverse order of IDs (i.e., bottommost op is first)
std::sort(parallel_ops_ids_.begin(), parallel_ops_ids_.end(), std::greater<>());
leaf_op_id_ = ops_.size() - 1;
if (parallel_ops_ids_.size() != 0) {
CHECK_FAIL_RETURN_UNEXPECTED(parallel_ops_ids_[parallel_ops_ids_.size() - 1] != 0,
"Non-sink pipeline, root node is a ParallelOp. Dataset AutoTune is not supported.");
}
return Status::OK();
}
@ -220,20 +214,7 @@ double AutoTune::Mean(const std::vector<T> &items) {
return std::accumulate(items.begin(), items.end(), 0.0) / static_cast<double>(items.size());
}
Status IsSinkCheck(bool sink_type) {
// Close AutoTune in Non-sink mode, since it's not ready for test.
if (sink_type == true) {
return Status::OK();
} else {
MS_LOG(ERROR) << "Dataset AutoTune doesn't support non-sink pipeline.";
return Status(StatusCode::kMDUnexpectedError,
"Dataset AutoTune hasn't been supported in non-sink mode(dataset_sink_mode=False), check training "
"config or set dataset_sink_mode to True.");
}
}
Status AutoTune::RunIterationEpoch() {
RETURN_IF_NOT_OK(IsSinkCheck(IsSink()));
// Run every epoch
if ((profiling_manager_->GetNumOfProfiledEpochs()) >= cur_epoch_) {
MS_LOG(INFO) << "Run Dataset AutoTune at epoch #" << cur_epoch_;
@ -244,7 +225,6 @@ Status AutoTune::RunIterationEpoch() {
}
Status AutoTune::RunIterationStep() {
RETURN_IF_NOT_OK(IsSinkCheck(IsSink()));
// Run at autotune step interval
int32_t step_temp = 0;
profiling_manager_->GetNumberOfProfiledSteps(&step_temp);

73
tests/ut/python/dataset/test_autotune.py
View File

@ -15,16 +15,22 @@
"""
Testing Autotune support in DE
"""
import sys
import numpy as np
import pytest
import mindspore._c_dataengine as cde
import mindspore.dataset as ds
def err_out_log(out, err, log=False):
if log:
sys.stdout.write(out)
sys.stderr.write(err)
# pylint: disable=unused-variable
@pytest.mark.forked
class TestAutotuneWithProfiler:
def test_autotune_after_profiler_with_1_pipeline(self, capfd):
@staticmethod
def test_autotune_after_profiler_with_1_pipeline(capfd):
"""
Feature: Autotuning with Profiler
Description: Test Autotune enabled together with MD Profiler with a single pipeline
@ -40,17 +46,18 @@ class TestAutotuneWithProfiler:
data1 = data1.batch(32)
itr1 = data1.create_dict_iterator(num_epochs=5)
_, err = capfd.readouterr()
out, err = capfd.readouterr()
assert "Cannot enable AutoTune for the current data pipeline as Dataset Profiling is already enabled for the " \
"current data pipeline." in err
# Uncomment the following two lines to see complete stdout and stderr output in pytest summary output
# sys.stdout.write(_)
# sys.stderr.write(err)
# Change False to True in the following call to see complete stdout and stderr output in pytest summary output
err_out_log(out, err, False)
md_profiler.stop()
ds.config.set_enable_autotune(False)
def test_autotune_after_profiler_with_2_pipeline(self, capfd):
@staticmethod
def test_autotune_after_profiler_with_2_pipeline(capfd):
"""
Feature: Autotuning with Profiler
Description: Test Autotune enabled together with MD Profiler with two pipelines
@ -68,28 +75,26 @@ class TestAutotuneWithProfiler:
data1 = data1.batch(32)
itr1 = data1.create_dict_iterator(num_epochs=5)
_, err = capfd.readouterr()
out, err = capfd.readouterr()
assert "Cannot enable AutoTune for the current data pipeline as Dataset Profiling is already enabled for the " \
"current data pipeline." in err
# Uncomment the following two lines to see complete stdout and stderr output in pytest summary output
# sys.stdout.write(_)
# sys.stderr.write(err)
# Change False to True in the following call to see complete stdout and stderr output in pytest summary output
err_out_log(out, err, False)
itr2 = data1.create_dict_iterator(num_epochs=5)
_, err = capfd.readouterr()
out, err = capfd.readouterr()
assert "Dataset Profiling is already enabled for a different data pipeline." in err
assert "Cannot enable AutoTune for the current data pipeline as Dataset Profiling is enabled for another data" \
" pipeline." in err
# Uncomment the following two lines to see complete stdout and stderr output in pytest summary output
# sys.stdout.write(_)
# sys.stderr.write(err)
# Change False to True in the following call to see complete stdout and stderr output in pytest summary output
err_out_log(out, err, False)
md_profiler.stop()
ds.config.set_enable_autotune(False)
@pytest.mark.skip(reason="close non-sink")
def test_autotune_with_2_pipeline(self, capfd):
@staticmethod
def test_autotune_with_2_pipeline(capfd):
"""
Feature: Autotuning
Description: Test Autotune two pipelines
@ -104,17 +109,16 @@ class TestAutotuneWithProfiler:
itr1 = data1.create_dict_iterator(num_epochs=5)
itr2 = data1.create_dict_iterator(num_epochs=5)
_, err = capfd.readouterr()
out, err = capfd.readouterr()
assert "Cannot enable AutoTune for the current data pipeline as it is already enabled for another data " \
"pipeline." in err
# Uncomment the following two lines to see complete stdout and stderr output in pytest summary output
# sys.stdout.write(_)
# sys.stderr.write(err)
# Change False to True in the following call to see complete stdout and stderr output in pytest summary output
err_out_log(out, err, False)
ds.config.set_enable_autotune(False)
@pytest.mark.skip(reason="close non-sink")
def test_delayed_autotune_with_2_pipeline(self, capfd):
@staticmethod
def test_delayed_autotune_with_2_pipeline(capfd):
"""
Feature: Autotuning
Description: Test delayed Autotune with two pipelines
@ -130,14 +134,13 @@ class TestAutotuneWithProfiler:
itr2 = data1.create_dict_iterator(num_epochs=5)
ds.config.set_enable_autotune(False)
_, err = capfd.readouterr()
out, err = capfd.readouterr()
assert err == ''
# Uncomment the following two lines to see complete stdout and stderr output in pytest summary output
# sys.stdout.write(_)
# sys.stderr.write(err)
# Change False to True in the following call to see complete stdout and stderr output in pytest summary output
err_out_log(out, err, False)
@pytest.mark.skip(reason="close non-sink")
def test_delayed_start_autotune_with_3_pipeline(self, capfd):
@staticmethod
def test_delayed_start_autotune_with_3_pipeline(capfd):
"""
Feature: Autotuning
Description: Test delayed Autotune and early stop with three pipelines
@ -155,14 +158,13 @@ class TestAutotuneWithProfiler:
itr3 = data1.create_dict_iterator(num_epochs=5)
_, err = capfd.readouterr()
out, err = capfd.readouterr()
assert err == ''
# Uncomment the following two lines to see complete stdout and stderr output in pytest summary output
# sys.stdout.write(_)
# sys.stderr.write(err)
# Change False to True in the following call to see complete stdout and stderr output in pytest summary output
err_out_log(out, err, False)
@pytest.mark.skip(reason="close non-sink")
def test_autotune_before_profiler(self):
@staticmethod
def test_autotune_before_profiler():
"""
Feature: Autotuning with Profiler
Description: Test Autotune with Profiler when Profiler is Initialized after autotune
@ -184,7 +186,8 @@ class TestAutotuneWithProfiler:
assert "Unexpected error. Stop MD Autotune before initializing the MD Profiler." in str(excinfo.value)
def test_autotune_simple_pipeline(self):
@staticmethod
def test_autotune_simple_pipeline():
"""
Feature: Autotuning
Description: test simple pipeline of autotune - Generator -> Shuffle -> Batch