Nigel
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

!29700 [MD] Tidy up UT and device queue info messages 

Merge pull request !29700 from cathwong/ckw_device_queue_probes
This commit is contained in:
i-robot 2022-02-09 12:47:27 +00:00 committed by Gitee
parent d0877dda3c fb469f1ad2
commit 5858a06689
No known key found for this signature in database
GPG Key ID: 173E9B9CA92EEF8F
4 changed files with 15 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
mindspore/ccsrc/minddata/dataset/engine/datasetops/device_queue_op.cc
View File

@ -1,5 +1,5 @@
/**
* Copyright 2019-2021 Huawei Technologies Co., Ltd
* Copyright 2019-2022 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -285,7 +285,7 @@ Status DeviceQueueOp::SendDataToAscend() {
send_finished_ = true;
}
tree_->SetFinished();
MS_LOG(INFO) << "Device queue send " << send_batch << " batch.";
MS_LOG(INFO) << "ExecutionTree finished. Device queue sent number of batches: " << send_batch;
return Status::OK();
}
@ -505,7 +505,7 @@ Status DeviceQueueOp::PushDataToGPU() {
send_finished_ = true;
}
tree_->SetFinished();
MS_LOG(INFO) << "Device queue send " << send_batch << " batch.";
MS_LOG(INFO) << "ExecutionTree finished. Device queue pushed number of batches: " << send_batch;
GpuBufferMgr::GetInstance().Close(handle);
GpuBufferMgr::GetInstance().CloseConfirm();
@ -581,7 +581,7 @@ Status DeviceQueueOp::WorkerEntry(int32_t worker_id) {
RETURN_IF_NOT_OK(receive_queues_[worker_id]->PopFront(&current_row));
}
MS_LOG(INFO) << "Device queue worker id " << worker_id << "proc " << batch_num << "batch.";
MS_LOG(INFO) << "Device queue worker id " << worker_id << " processed number of batches: " << batch_num;
// Add empty data_item vector with eoe_flag=false as quit flag.
GpuConnectorItem connector_item = {{}, false};
RETURN_IF_NOT_OK(gpu_connector_->Add(worker_id, std::move(connector_item)));
@ -641,7 +641,7 @@ Status DeviceQueueOp::SendDataToGPU() {
RETURN_IF_NOT_OK(receive_queues_[num_buf++ % num_workers_]->Add(std::move(quit_flag)));
}
MS_LOG(INFO) << "Device queue receive " << num_buf - num_workers_ << " batch.";
MS_LOG(INFO) << "Device queue received number of batches and EOEs: " << (num_buf - num_workers_);
return Status::OK();
}

8
tests/ut/python/dataset/test_batch.py
View File

@ -1,4 +1,4 @@
# Copyright 2019 Huawei Technologies Co., Ltd
# Copyright 2019-2022 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -240,9 +240,11 @@ def test_batch_12():
def test_batch_13():
"""
Test batch: python_multiprocessing is True and does not work for per_batch_map is None
Feature: Batch op
Description: Test python_multiprocessing is True with per_batch_map is None
Expectation: python_multiprocessing is True is ignored when per_batch_map is None
"""
logger.info("test_batch_12")
logger.info("test_batch_13")
# define parameters
batch_size = True

3
tests/ut/python/dataset/test_datasets_textfileop.py
View File

@ -1,4 +1,4 @@
# Copyright 2020 Huawei Technologies Co., Ltd
# Copyright 2020-2022 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -268,6 +268,7 @@ if __name__ == "__main__":
test_textline_dataset_num_samples()
test_textline_dataset_distribution()
test_textline_dataset_repeat()
test_textline_dataset_output_tensor()
test_textline_dataset_get_datasetsize()
test_textline_dataset_to_device()
test_textline_dataset_exceptions()

6
tests/ut/python/dataset/test_sync_wait.py
View File

@ -1,4 +1,5 @@
# Copyright 2019 Huawei Technologies Co., Ltd
# Copyright 2019i-2022 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -232,14 +233,13 @@ def test_sync_exception_04():
def test_sync_exception_05():
"""
Test sync: with wrong batch size in update
Test sync: with wrong condition name in update
"""
logger.info("test_sync_exception_05")
dataset = ds.GeneratorDataset(gen, column_names=["input"])
count = 0
aug = Augment(0)
# try to create dataset with batch_size < 0
dataset = dataset.sync_wait(condition_name="every batch", callback=aug.update)
dataset = dataset.map(operations=[aug.preprocess], input_columns=["input"])
with pytest.raises(RuntimeError) as e: