!18416 [MD] fix some bugs in master

Merge pull request !18416 from liyong126/fix_some_bugs

mindspore/ccsrc/minddata/dataset/engine/consumers/tree_consumer.cc

@@ -347,7 +347,8 @@ Status SaveToDisk::CheckTensorRowShapes(const std::unordered_map<std::string, in
     return Status::OK();
   }
   auto res = map_compare(*PreTensorRowShapes_ptr, CurrTensorRowShapes);
-  CHECK_FAIL_RETURN_UNEXPECTED(res, "Error: current tensor shape is different from the previous's.");
+  CHECK_FAIL_RETURN_UNEXPECTED(res,
+                               "Error: besides dimension 0, other dimension shape is different from the previous's.");
   return Status::OK();
 }
 

mindspore/mindrecord/common/exceptions.py

@@ -100,8 +100,7 @@ class MindRecordException(Exception):
         return self._error_code
 
     def __str__(self):
-        return "[{}]: error_code: {}, error_msg: {}".format(
-            self.__class__.__name__, self._error_code, self._error_msg)
+        return "[{}]: {}".format(self.__class__.__name__, self._error_msg)
 
     @staticmethod
     def transform_error_code(exception_no):

mindspore/mindrecord/tools/cifar10.py

@@ -64,7 +64,9 @@ def restricted_loads(s):
         return RestrictedUnpickler(f, encoding='bytes').load()
     except pickle.UnpicklingError:
         raise RuntimeError("Not a valid Cifar10 Dataset.")
-    else:
+    except UnicodeDecodeError:
+        raise RuntimeError("Not a valid Cifar10 Dataset.")
+    except:
         raise RuntimeError("Unexpected error while Unpickling Cifar10 Dataset.")
 
 

mindspore/mindrecord/tools/cifar100.py

@@ -63,7 +63,9 @@ def restricted_loads(s):
         return RestrictedUnpickler(f, encoding='bytes').load()
     except pickle.UnpicklingError:
         raise RuntimeError("Not a valid Cifar100 Dataset.")
-    else:
+    except UnicodeDecodeError:
+        raise RuntimeError("Not a valid Cifar100 Dataset.")
+    except:
         raise RuntimeError("Unexpected error while Unpickling Cifar100 Dataset.")
 
 

tests/ut/python/dataset/test_save_op.py

@@ -322,7 +322,8 @@ def test_case_03(add_remove_file):
                         shuffle=False)
 
     i = 0
-    for item in d2.create_dict_iterator(num_epochs=1, output_numpy=True):  # each data is a dictionary
+    # each data is a dictionary
+    for item in d2.create_dict_iterator(num_epochs=1, output_numpy=True):
         golden = np.array([i])
         np.testing.assert_array_equal(item["data"], golden)
         i = i + 1
@@ -351,7 +352,8 @@ def type_tester(t):
 
     i = 0
     num_repeat = 0
-    for item in d2.create_dict_iterator(num_epochs=1, output_numpy=True):  # each data is a dictionary
+    # each data is a dictionary
+    for item in d2.create_dict_iterator(num_epochs=1, output_numpy=True):
         golden = np.array([[i], [i + 1], [i + 2], [i + 3]], dtype=t)
         logger.info(item)
         np.testing.assert_array_equal(item["data"], golden)
@@ -433,6 +435,7 @@ def test_case_07():
     if os.path.exists("{}.db".format(AUTO_FILE)):
         os.remove("{}.db".format(AUTO_FILE))
 
+
 def generator_dynamic_1d():
     arr = []
     for i in range(10):
@@ -441,6 +444,7 @@ def generator_dynamic_1d():
         arr += [i]
         yield (np.array(arr),)
 
+
 def generator_dynamic_2d_0():
     for i in range(10):
         if i < 5:
@@ -456,6 +460,7 @@ def generator_dynamic_2d_1():
         else:
             yield (np.arange(10).reshape([5, 2]),)
 
+
 def test_case_08(add_remove_file):
 
     # apply dataset operations
@@ -477,6 +482,7 @@ def test_case_08(add_remove_file):
         np.testing.assert_array_equal(item["data"], golden)
         i = i + 1
 
+
 def test_case_09(add_remove_file):
 
     # apply dataset operations
@@ -497,10 +503,12 @@ def test_case_09(add_remove_file):
         np.testing.assert_array_equal(item["data"], golden)
         i = i + 1
 
+
 def test_case_10(add_remove_file):
 
     # apply dataset operations
     d1 = ds.GeneratorDataset(generator_dynamic_2d_1, ["data"], shuffle=False)
 
-    with pytest.raises(Exception, match="Error: current tensor shape is different from the previous's"):
+    with pytest.raises(Exception, match=
+                       "Error: besides dimension 0, other dimension shape is different from the previous's"):
         d1.save(AUTO_FILE)

tests/ut/python/mindrecord/test_mindrecord_exception.py

@@ -109,8 +109,7 @@ def test_lack_partition_and_db():
     with pytest.raises(MRMOpenError) as err:
         reader = FileReader('dummy.mindrecord')
         reader.close()
-    assert '[MRMOpenError]: error_code: 1347690596, ' \
-           'error_msg: MindRecord File could not open successfully.' \
+    assert '[MRMOpenError]: MindRecord File could not open successfully.' \
            in str(err.value)
 
 def test_lack_db(fixture_cv_file):
@@ -120,8 +119,7 @@ def test_lack_db(fixture_cv_file):
     with pytest.raises(MRMOpenError) as err:
         reader = FileReader(CV_FILE_NAME)
         reader.close()
-    assert '[MRMOpenError]: error_code: 1347690596, ' \
-           'error_msg: MindRecord File could not open successfully.' \
+    assert '[MRMOpenError]: MindRecord File could not open successfully.' \
            in str(err.value)
 
 def test_lack_some_partition_and_db(fixture_cv_file):
@@ -134,8 +132,7 @@ def test_lack_some_partition_and_db(fixture_cv_file):
     with pytest.raises(MRMOpenError) as err:
         reader = FileReader(CV_FILE_NAME + "0")
         reader.close()
-    assert '[MRMOpenError]: error_code: 1347690596, ' \
-           'error_msg: MindRecord File could not open successfully.' \
+    assert '[MRMOpenError]: MindRecord File could not open successfully.' \
            in str(err.value)
 
 def test_lack_some_partition_first(fixture_cv_file):
@@ -147,8 +144,7 @@ def test_lack_some_partition_first(fixture_cv_file):
     with pytest.raises(MRMOpenError) as err:
         reader = FileReader(CV_FILE_NAME + "0")
         reader.close()
-    assert '[MRMOpenError]: error_code: 1347690596, ' \
-           'error_msg: MindRecord File could not open successfully.' \
+    assert '[MRMOpenError]: MindRecord File could not open successfully.' \
            in str(err.value)
 
 def test_lack_some_partition_middle(fixture_cv_file):
@@ -160,8 +156,7 @@ def test_lack_some_partition_middle(fixture_cv_file):
     with pytest.raises(MRMOpenError) as err:
         reader = FileReader(CV_FILE_NAME + "0")
         reader.close()
-    assert '[MRMOpenError]: error_code: 1347690596, ' \
-           'error_msg: MindRecord File could not open successfully.' \
+    assert '[MRMOpenError]: MindRecord File could not open successfully.' \
            in str(err.value)
 
 def test_lack_some_partition_last(fixture_cv_file):
@@ -173,8 +168,7 @@ def test_lack_some_partition_last(fixture_cv_file):
     with pytest.raises(MRMOpenError) as err:
         reader = FileReader(CV_FILE_NAME + "0")
         reader.close()
-    assert '[MRMOpenError]: error_code: 1347690596, ' \
-           'error_msg: MindRecord File could not open successfully.' \
+    assert '[MRMOpenError]: MindRecord File could not open successfully.' \
            in str(err.value)
 
 def test_mindpage_lack_some_partition(fixture_cv_file):
@@ -185,8 +179,7 @@ def test_mindpage_lack_some_partition(fixture_cv_file):
     os.remove("{}".format(paths[0]))
     with pytest.raises(MRMOpenError) as err:
         MindPage(CV_FILE_NAME + "0")
-    assert '[MRMOpenError]: error_code: 1347690596, ' \
-           'error_msg: MindRecord File could not open successfully.' \
+    assert '[MRMOpenError]: MindRecord File could not open successfully.' \
            in str(err.value)
 
 def test_lack_some_db(fixture_cv_file):
@@ -198,8 +191,7 @@ def test_lack_some_db(fixture_cv_file):
     with pytest.raises(MRMOpenError) as err:
         reader = FileReader(CV_FILE_NAME + "0")
         reader.close()
-    assert '[MRMOpenError]: error_code: 1347690596, ' \
-           'error_msg: MindRecord File could not open successfully.' \
+    assert '[MRMOpenError]: MindRecord File could not open successfully.' \
            in str(err.value)
 
 
@@ -210,8 +202,7 @@ def test_invalid_mindrecord():
         f.write(dummy)
     with pytest.raises(MRMOpenError) as err:
         FileReader(CV_FILE_NAME)
-    assert '[MRMOpenError]: error_code: 1347690596, ' \
-           'error_msg: MindRecord File could not open successfully.' \
+    assert '[MRMOpenError]: MindRecord File could not open successfully.' \
            in str(err.value)
     os.remove(CV_FILE_NAME)
 
@@ -223,8 +214,7 @@ def test_invalid_db(fixture_cv_file):
         f.write('just for test')
     with pytest.raises(MRMOpenError) as err:
         FileReader('imagenet.mindrecord')
-    assert '[MRMOpenError]: error_code: 1347690596, ' \
-           'error_msg: MindRecord File could not open successfully.' \
+    assert '[MRMOpenError]: MindRecord File could not open successfully.' \
            in str(err.value)
 
 def test_overwrite_invalid_mindrecord(fixture_cv_file):
@@ -233,8 +223,7 @@ def test_overwrite_invalid_mindrecord(fixture_cv_file):
         f.write('just for test')
     with pytest.raises(MRMOpenError) as err:
         create_cv_mindrecord(1)
-    assert '[MRMOpenError]: error_code: 1347690596, ' \
-           'error_msg: MindRecord File could not open successfully.' \
+    assert '[MRMOpenError]: MindRecord File could not open successfully.' \
            in str(err.value)
 
 def test_overwrite_invalid_db(fixture_cv_file):
@@ -243,8 +232,7 @@ def test_overwrite_invalid_db(fixture_cv_file):
         f.write('just for test')
     with pytest.raises(MRMGenerateIndexError) as err:
         create_cv_mindrecord(1)
-    assert '[MRMGenerateIndexError]: error_code: 1347690612, ' \
-           'error_msg: Failed to generate index.' in str(err.value)
+    assert '[MRMGenerateIndexError]: Failed to generate index.' in str(err.value)
 
 def test_read_after_close(fixture_cv_file):
     """test file reader when close read."""