mindspore-Ecosystem
/
mindspore
21
0
Fork 27
Code Issues Pull Requests 6 Projects Releases Wiki Activity

!139 remove ENABLE_MINDRECORD flag 

Merge pull request !139 from jonathan_yan/DLTJ
This commit is contained in:
mindspore-ci-bot 2020-04-08 08:19:32 +08:00 committed by Gitee
parent 9aab1613e7 243120bfa3
commit 993e69c76e
8 changed files with 3 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
mindspore/ccsrc/dataset/CMakeLists.txt
View File

@ -17,8 +17,6 @@ if (ENABLE_TDTQUE)
message(STATUS "TDT queue is enabled")
endif ()
add_definitions(-D ENABLE_MINDRECORD)
# conde coverage
# option(ENABLE_COVERAGE "Enable code coverage report" OFF)
# if (ENABLE_COVERAGE)

12
mindspore/ccsrc/dataset/api/de_pipeline.cc
View File

@ -29,11 +29,9 @@
#include "dataset/engine/datasetops/source/manifest_op.h"
#include "dataset/engine/datasetops/source/cifar_op.h"
#include "dataset/engine/datasetops/source/celeba_op.h"
#ifdef ENABLE_MINDRECORD
#include "./shard_category.h"
#include "./shard_sample.h"
#include "./shard_shuffle.h"
#endif
#include "mindrecord/include/shard_category.h"
#include "mindrecord/include/shard_sample.h"
#include "mindrecord/include/shard_shuffle.h"
#include "dataset/util/random.h"
#include "dataset/util/status.h"
@ -46,9 +44,7 @@ using pFunction = Status (DEPipeline::*)(const py::dict &, std::shared_ptr<Datas
static std::unordered_map<uint32_t, pFunction> g_parse_op_func_ = {{kStorage, &DEPipeline::ParseStorageOp},
{kShuffle, &DEPipeline::ParseShuffleOp},
#ifdef ENABLE_MINDRECORD
{kMindrecord, &DEPipeline::ParseMindRecordOp},
#endif
{kMap, &DEPipeline::ParseMapOp},
{kBatch, &DEPipeline::ParseBatchOp},
{kRepeat, &DEPipeline::ParseRepeatOp},
@ -364,7 +360,6 @@ Status DEPipeline::ParseShuffleOp(const py::dict &args, std::shared_ptr<DatasetO
return Status::OK();
}
#ifdef ENABLE_MINDRECORD
Status DEPipeline::CheckMindRecordPartitionInfo(const py::dict &args, std::vector<int> *in_partitions) {
if (args["partitions"].is_none()) {
std::string err_msg = "Error: partitions is not set (None)";
@ -450,7 +445,6 @@ Status DEPipeline::ParseMindRecordOp(const py::dict &args, std::shared_ptr<Datas
*ptr = op;
return Status::OK();
}
#endif
Status DEPipeline::ParseMapOp(const py::dict &args, std::shared_ptr<DatasetOp> *ptr) {
std::shared_ptr<MapOp::Builder> builder = std::make_shared<MapOp::Builder>();

4
mindspore/ccsrc/dataset/api/de_pipeline.h
View File

@ -38,9 +38,7 @@ using DsOpPtr = std::shared_ptr<DatasetOp>;
enum OpName {
kStorage = 0,
kShuffle,
#ifdef ENABLE_MINDRECORD
kMindrecord,
#endif
kBatch,
kCache,
kRepeat,
@ -101,11 +99,9 @@ class DEPipeline {
Status ParseShuffleOp(const py::dict &args, std::shared_ptr<DatasetOp> *ptr);
#ifdef ENABLE_MINDRECORD
Status CheckMindRecordPartitionInfo(const py::dict &args, std::vector<int> *ptr);
Status ParseMindRecordOp(const py::dict &args, std::shared_ptr<DatasetOp> *ptr);
#endif
Status ParseMapOp(const py::dict &args, std::shared_ptr<DatasetOp> *ptr);

6
mindspore/ccsrc/dataset/api/python_bindings.cc
View File

@ -44,9 +44,7 @@
#include "dataset/engine/datasetops/source/io_block.h"
#include "dataset/engine/datasetops/source/mnist_op.h"
#include "dataset/engine/datasetops/source/manifest_op.h"
#ifdef ENABLE_MINDRECORD
#include "dataset/engine/datasetops/source/mindrecord_op.h"
#endif
#include "dataset/engine/datasetops/source/sampler/distributed_sampler.h"
#include "dataset/engine/datasetops/source/sampler/pk_sampler.h"
#include "dataset/engine/datasetops/source/sampler/random_sampler.h"
@ -146,14 +144,12 @@ void bindDatasetOps(py::module *m) {
return py::make_tuple(count, num_classes);
});
#ifdef ENABLE_MINDRECORD
(void)py::class_<MindRecordOp, DatasetOp, std::shared_ptr<MindRecordOp>>(*m, "MindRecordOp")
.def_static("get_num_rows", [](const std::string &path) {
int64_t count = 0;
THROW_IF_ERROR(MindRecordOp::CountTotalRows(path, &count));
return count;
});
#endif
(void)py::class_<ManifestOp, DatasetOp, std::shared_ptr<ManifestOp>>(*m, "ManifestOp")
.def_static("get_num_rows_and_classes",
@ -424,9 +420,7 @@ PYBIND11_MODULE(_c_dataengine, m) {
.value("STORAGE", OpName::kStorage)
.value("SHUFFLE", OpName::kShuffle)
.value("BATCH", OpName::kBatch)
#ifdef ENABLE_MINDRECORD
.value("MINDRECORD", OpName::kMindrecord)
#endif
.value("CACHE", OpName::kCache)
.value("REPEAT", OpName::kRepeat)
.value("TAKE", OpName::kTake)

3
mindspore/ccsrc/dataset/engine/datasetops/source/mindrecord_op.cc
View File

@ -13,8 +13,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifdef ENABLE_MINDRECORD
#include "dataset/engine/datasetops/source/mindrecord_op.h"
#include <algorithm>
@ -665,4 +663,3 @@ Status MindRecordOp::CountTotalRows(const std::string dataset_path, int64_t *cou
}
} // namespace dataset
} // namespace mindspore
#endif

2
mindspore/ccsrc/dataset/engine/datasetops/source/mindrecord_op.h
View File

@ -15,7 +15,6 @@
*/
#ifndef DATASET_ENGINE_DATASETOPS_SOURCE_MINDRECORD_OP_H_
#define DATASET_ENGINE_DATASETOPS_SOURCE_MINDRECORD_OP_H_
#ifdef ENABLE_MINDRECORD
#pragma once
#include <cstdint>
@ -276,5 +275,4 @@ class MindRecordOp : public ParallelOp {
};
} // namespace dataset
} // namespace mindspore
#endif
#endif // DATASET_ENGINE_DATASETOPS_SOURCE_MINDRECORD_OP_H_

1
tests/ut/cpp/CMakeLists.txt
View File

@ -26,7 +26,6 @@ MESSAGE("check ut_test ${CMAKE_BINARY_DIR}")
link_directories(${MS_CCSRC_BUILD_PATH})
if(ENABLE_MINDDATA)
add_definitions(-D ENABLE_MINDRECORD)
add_definitions(-D ENABLE_MINDDATA)
link_directories(${MS_CCSRC_BUILD_PATH}/dataset)
link_directories(${MS_CCSRC_BUILD_PATH}/mindrecord)

2
tests/ut/cpp/dataset/mind_record_op_test.cc
View File

@ -13,7 +13,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifdef ENABLE_MINDRECORD
#include <iostream>
#include <memory>
#include <vector>
@ -480,4 +479,3 @@ TEST_F(MindDataTestMindRecordOp, TestMindRecordBlockReaderRepeat) {
row_count++;
}
}
#endif