p60975123
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

!17507 [MD] fix codecheck in master 

From: @liyong126
Reviewed-by: @heleiwang,@jonyguo
Signed-off-by: @jonyguo
This commit is contained in:
mindspore-ci-bot 2021-06-04 09:47:37 +08:00 committed by Gitee
parent d79d92bfbf 487c330985
commit fb5eea169b
5 changed files with 22 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
mindspore/ccsrc/minddata/dataset/kernels/image/soft_dvpp/utils/external_soft_dp.h
View File

@ -17,7 +17,7 @@
#ifndef EXTERNAL_SOFTDP_H
#define EXTERNAL_SOFTDP_H
#include <cstdint>
#include <stdint.h>
struct SoftDpProcsessInfo {
uint8_t *input_buffer; // input buffer

1
mindspore/ccsrc/minddata/mindrecord/io/shard_reader.cc
View File

@ -636,7 +636,6 @@ std::pair<MSRStatus, std::vector<uint64_t>> ShardReader::GetPagesByCategory(
if (!criteria.first.empty()) {
auto schema = shard_header_->GetSchemas()[0]->GetSchema();
if (kNumberFieldTypeSet.find(schema["schema"][criteria.first]["type"]) != kNumberFieldTypeSet.end()) {
sql +=
" AND " + criteria.first + "_" + std::to_string(column_schema_id_[criteria.first]) + " = " + criteria.second;

2
mindspore/ccsrc/minddata/mindrecord/io/shard_writer.cc
View File

@ -660,7 +660,7 @@ MSRStatus ShardWriter::MergeBlobData(const std::vector<string> &blob_fields,
uint64_t blob_size = b->size();
// big edian
for (size_t i = 0; i < buf.size(); ++i) {
buf[buf.size() - 1 - i] = (std::numeric_limits<uint8_t>::max() & blob_size);
buf[buf.size() - 1 - i] = (std::numeric_limits<uint8_t>::max()) & blob_size;
blob_size >>= 8u;
}
std::copy(buf.begin(), buf.end(), (*output)->begin() + idx);

13
mindspore/mindrecord/shardutils.py
View File

@ -125,13 +125,12 @@ def check_parameter(func):
else:
check_filename(value)
if name == 'num_consumer':
if value is not None:
if isinstance(value, int):
if value < MIN_CONSUMER_COUNT or value > MAX_CONSUMER_COUNT():
raise ParamValueError("Consumer number should between {} and {}."
.format(MIN_CONSUMER_COUNT, MAX_CONSUMER_COUNT()))
else:
raise ParamValueError("Consumer number is illegal.")
if value is None:
raise ParamValueError("Consumer number is illegal.")
if isinstance(value, int):
if value < MIN_CONSUMER_COUNT or value > MAX_CONSUMER_COUNT():
raise ParamValueError("Consumer number should between {} and {}."
.format(MIN_CONSUMER_COUNT, MAX_CONSUMER_COUNT()))
else:
raise ParamValueError("Consumer number is illegal.")
return func(*args, **kw)

34
mindspore/train/model.py
View File

@ -282,9 +282,15 @@ class Model:
scaling_sens /= self._device_number
return scaling_sens
def _exec_preprocess(self, network, is_train, phase, dataset,
dataset_sink_mode, sink_size=-1, epoch_num=1, dataset_helper=None):
def _exec_preprocess(self, is_train, dataset, dataset_sink_mode, sink_size=-1, epoch_num=1, dataset_helper=None):
"""Initializes dataset."""
if is_train:
network = self._train_network
phase = 'train'
else:
network = self._eval_network
phase = 'eval'
if dataset_sink_mode and not is_train:
dataset.__loop_size__ = 1
@ -330,9 +336,7 @@ class Model:
self._train_network.set_broadcast_flag()
train_dataset.__no_send__ = True
train_dataset_helper, train_network = self._exec_preprocess(self._train_network,
is_train=True,
phase='train',
train_dataset_helper, train_network = self._exec_preprocess(is_train=True,
dataset=train_dataset,
dataset_sink_mode=True,
sink_size=sink_size)
@ -346,9 +350,7 @@ class Model:
raise RuntimeError('If define `valid_dataset`, metric fn can not be None or empty.')
valid_dataset.__no_send__ = True
valid_dataset_helper, eval_network = self._exec_preprocess(self._eval_network,
is_train=False,
phase='eval',
valid_dataset_helper, eval_network = self._exec_preprocess(is_train=False,
dataset=valid_dataset,
dataset_sink_mode=True)
self._eval_network = eval_network
@ -456,9 +458,7 @@ class Model:
for i in range(epoch):
cb_params.cur_epoch_num = i + 1
list_callback.epoch_begin(run_context)
dataset_helper, train_network = self._exec_preprocess(self._train_network,
is_train=True,
phase='train',
dataset_helper, train_network = self._exec_preprocess(is_train=True,
dataset=train_dataset,
dataset_sink_mode=True,
sink_size=sink_size,
@ -506,9 +506,7 @@ class Model:
list_callback (Callback): Executor of callback list. Default: None.
cb_params (_InternalCallbackParam): Callback parameters. Default: None.
"""
dataset_helper, _ = self._exec_preprocess(self._train_network,
is_train=True,
phase='train',
dataset_helper, _ = self._exec_preprocess(is_train=True,
dataset=train_dataset,
dataset_sink_mode=False,
epoch_num=epoch)
@ -640,9 +638,7 @@ class Model:
"""
run_context = RunContext(cb_params)
dataset_helper, eval_network = self._exec_preprocess(self._eval_network,
is_train=False,
phase='eval',
dataset_helper, eval_network = self._exec_preprocess(is_train=False,
dataset=valid_dataset,
dataset_sink_mode=True)
self._eval_network = eval_network
@ -680,9 +676,7 @@ class Model:
run_context = RunContext(cb_params)
cb_params.dataset_sink_mode = False
list_callback.begin(run_context)
dataset_helper, _ = self._exec_preprocess(self._eval_network,
is_train=False,
phase='eval',
dataset_helper, _ = self._exec_preprocess(is_train=False,
dataset=valid_dataset,
dataset_sink_mode=False)
for next_element in dataset_helper: