From 97d86730186a0d5fd447bcb70e2b968a4cd439e0 Mon Sep 17 00:00:00 2001 From: Li Hongzhang Date: Wed, 24 Jun 2020 16:40:49 +0800 Subject: [PATCH] warn when values duplicate and set mode to 'eval' to avoid extra recording --- mindspore/train/callback/_summary_collector.py | 2 ++ mindspore/train/summary/_writer_pool.py | 2 +- mindspore/train/summary/summary_record.py | 3 +++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/mindspore/train/callback/_summary_collector.py b/mindspore/train/callback/_summary_collector.py index cff03ca398..41bda092a5 100644 --- a/mindspore/train/callback/_summary_collector.py +++ b/mindspore/train/callback/_summary_collector.py @@ -308,6 +308,8 @@ class SummaryCollector(Callback): else: self._collect_eval_lineage(cb_params) + # This is a workaround to avoid record '_summary_tensor_cache'. + self._record.set_mode('eval') # There's nothing special about setting step to 0 here, just to satisfy the interface call self._record.record(step=0) diff --git a/mindspore/train/summary/_writer_pool.py b/mindspore/train/summary/_writer_pool.py index 2d219743de..f2ebca35cd 100644 --- a/mindspore/train/summary/_writer_pool.py +++ b/mindspore/train/summary/_writer_pool.py @@ -59,7 +59,7 @@ class WriterPool(Process): def run(self): writers = self._get_writers() - with Pool() as pool: + with Pool(min(cpu_count(), 32)) as pool: deq = deque() while True: while deq and deq[0].ready(): diff --git a/mindspore/train/summary/summary_record.py b/mindspore/train/summary/summary_record.py index 61c2c8adeb..bcb7334e7a 100644 --- a/mindspore/train/summary/summary_record.py +++ b/mindspore/train/summary/summary_record.py @@ -202,6 +202,9 @@ class SummaryRecord: if not isinstance(value, Tensor): raise TypeError(f'Expect the value to be Tensor, but got {type(value).__name__}') np_value = _check_to_numpy(plugin, value) + if name in {item['tag'] for item in self._data_pool[plugin]}: + entry = repr(f'{name}/{plugin}') + logger.warning(f'{entry} has duplicate values. Only the newest one will be recorded.') self._data_pool[plugin].append(dict(tag=name, mode=self._mode, value=np_value)) elif plugin in ('train_lineage', 'eval_lineage', 'dataset_graph', 'custom_lineage_data'):